diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties index 3cb2fedabb4..21231d2f508 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties @@ -37,6 +37,24 @@ che.workspace.auto_snapshot=true # Otherwise create a new workspace. che.workspace.auto_restore=true + +# Workspace threads pool configuration, this pool is used for workspace related +# operations that require asynchronous execution e.g. starting/stopping/snapshotting + +# possible values are 'fixed', 'cached' +che.workspace.pool.type=fixed + +# This property is ignored when pool type is different from 'fixed'. +# Configures the exact size of the pool, if it's set multiplier property is ignored. +# If this property is not set(0, < 0, NULL) then pool sized to number of cores, +#it can be modified within multiplier +che.workspace.pool.exact_size=NULL + +# This property is ignored when pool type is different from 'fixed' or exact pool size is set. +# If it's set the pool size will be N_CORES * multiplier +che.workspace.pool.cores_multiplier=2 + + # Java command line options used to start Che agent in workspace runtime che.workspace.java_opts=-Xms256m -Xmx2048m -Djava.security.egd=file:/dev/./urandom diff --git a/core/che-core-api-core/src/main/java/org/eclipse/che/api/core/util/CompositeLineConsumer.java b/core/che-core-api-core/src/main/java/org/eclipse/che/api/core/util/CompositeLineConsumer.java index fb0d695bfcb..cf2439e5d8e 100644 --- a/core/che-core-api-core/src/main/java/org/eclipse/che/api/core/util/CompositeLineConsumer.java +++ b/core/che-core-api-core/src/main/java/org/eclipse/che/api/core/util/CompositeLineConsumer.java @@ -33,7 +33,7 @@ public class CompositeLineConsumer implements LineConsumer { private static final Logger LOG = LoggerFactory.getLogger(CompositeLineConsumer.class); private final List lineConsumers; - private boolean isOpen; + private boolean isOpen; public CompositeLineConsumer(LineConsumer... lineConsumers) { this.lineConsumers = new CopyOnWriteArrayList<>(lineConsumers); @@ -70,7 +70,11 @@ public void writeLine(String line) { for (LineConsumer lineConsumer : lineConsumers) { try { lineConsumer.writeLine(line); - } catch (ConsumerAlreadyClosedException | ClosedByInterruptException e) { + } catch (ClosedByInterruptException interrupted) { + Thread.currentThread().interrupt(); + isOpen = false; + return; + } catch (ConsumerAlreadyClosedException e) { lineConsumers.remove(lineConsumer); // consumer is already closed, so we cannot write into it any more if (lineConsumers.size() == 0) { // if all consumers are closed then we can close this one isOpen = false; diff --git a/core/che-core-api-core/src/test/java/org/eclipse/che/api/core/util/CompositeLineConsumerTest.java b/core/che-core-api-core/src/test/java/org/eclipse/che/api/core/util/CompositeLineConsumerTest.java index e568b5cdc57..94998fae0cb 100644 --- a/core/che-core-api-core/src/test/java/org/eclipse/che/api/core/util/CompositeLineConsumerTest.java +++ b/core/che-core-api-core/src/test/java/org/eclipse/che/api/core/util/CompositeLineConsumerTest.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.testng.Assert.assertTrue; /** * @author Mykola Morhun @@ -49,7 +50,7 @@ public class CompositeLineConsumerTest { @BeforeMethod public void beforeMethod() throws Exception { - subConsumers = new LineConsumer[] { lineConsumer1, lineConsumer2, lineConsumer3 }; + subConsumers = new LineConsumer[] {lineConsumer1, lineConsumer2, lineConsumer3}; compositeLineConsumer = new CompositeLineConsumer(subConsumers); } @@ -86,8 +87,7 @@ public void shouldNotWriteIntoSubConsumersAfterClosingCompositeConsumer() throws public Object[][] subConsumersExceptions() { return new Throwable[][] { {new ConsumerAlreadyClosedException("Error")}, - {new ClosedByInterruptException()} - }; + }; } @Test(dataProvider = "subConsumersExceptions") @@ -132,7 +132,19 @@ public void shouldDoNothingOnWriteLineIfAllSubConsumersAreClosed() throws Except } } - private LineConsumer[] appendTo(LineConsumer[] base, LineConsumer... toAppend ) { + @Test + public void stopsWritingOnceInterrupted() throws Exception { + doThrow(new ClosedByInterruptException()).when(lineConsumer2).writeLine("test"); + + compositeLineConsumer.writeLine("test"); + + assertTrue(Thread.interrupted()); + verify(lineConsumer1).writeLine("test"); + verify(lineConsumer2).writeLine("test"); + verify(lineConsumer3, never()).writeLine("test"); + } + + private LineConsumer[] appendTo(LineConsumer[] base, LineConsumer... toAppend) { List allElements = new ArrayList<>(); allElements.addAll(Arrays.asList(base)); allElements.addAll(Arrays.asList(toAppend)); diff --git a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java index 626b1fd44ae..4f65d8806c8 100644 --- a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java +++ b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java @@ -28,8 +28,9 @@ public enum WorkspaceStatus { *

Workspace becomes starting only if it was {@link #STOPPED}. * The status map: * 

-     *  STOPPED -> STARTING -> RUNNING (normal behaviour)
-     *  STOPPED -> STARTING -> STOPPED (failed to start)
+     *  STOPPED -> STARTING -> RUNNING  (normal behaviour)
+     *  STOPPED -> STARTING -> STOPPED  (failed to start)
+     *  STOPPED -> STARTING -> STOPPING (explicitly stopped)
      *
*/ STARTING, @@ -61,10 +62,12 @@ public enum WorkspaceStatus { /** * Workspace considered as stopping if and only if its active environment is shutting down. * - *

Workspace is in stopping status only if it was in {@link #RUNNING} status before. + *

Workspace is in stopping status only if it was in {@link #RUNNING} or + * {@link #STARTING} status before. * The status map: * 

-     *  RUNNING -> STOPPING -> STOPPED (normal behaviour)/(error while stopping)
+     *  RUNNING  -> STOPPING -> STOPPED (normal behaviour)/(error while stopping)
+     *  STARTING -> STOPPING -> STOPPED (stopped while starting)
      *
*/ STOPPING, diff --git a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/StripedLocks.java b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/StripedLocks.java index 15ddcd8b874..c0bbb28fca4 100644 --- a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/StripedLocks.java +++ b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/StripedLocks.java @@ -12,34 +12,36 @@ import com.google.common.util.concurrent.Striped; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; /** * Helper class to use striped locks in try-with-resources construction. *

* Examples of usage: - * 

- *     StripedLocks stripedLocks = new StripedLocks(16);
- *     try (CloseableLock lock = stripedLocks.acquireWriteLock(myKey)) {
- *         syncedObject.write();
- *     }
+ * 
{@code
+ *  StripedLocks stripedLocks = new StripedLocks(16);
+ *  try (Unlocker u = stripedLocks.writeLock(myKey)) {
+ *      syncedObject.write();
+ *  }
  *
- *     try (CloseableLock lock = stripedLocks.acquireReadLock(myKey)) {
- *         syncedObject.read();
- *     }
+ *  try (Unlocker u = stripedLocks.readLock(myKey)) {
+ *      syncedObject.read();
+ *  }
  *
- *     try (CloseableLock lock = stripedLocks.acquireWriteAllLock(myKey)) {
- *         for (ObjectToSync objectToSync : allObjectsToSync) {
- *             objectToSync.write();
- *         }
- *     }
- * 

+ *  try (Unlocker u = stripedLocks.writeAllLock(myKey)) {
+ *      for (ObjectToSync objectToSync : allObjectsToSync) {
+ *          objectToSync.write();
+ *      }
+ *  }
+ * }
* * @author Alexander Garagatyi * @author Sergii Leschenko + * @author Yevhenii Voevodin */ -// TODO consider usage of plain map with locks instead of Guava's Striped public class StripedLocks { + private final Striped striped; public StripedLocks(int stripesCount) { @@ -49,75 +51,66 @@ public StripedLocks(int stripesCount) { /** * Acquire read lock for provided key. */ - public CloseableLock acquireReadLock(String key) { - return new ReadLock(key); + public Unlocker readLock(String key) { + Lock lock = striped.get(key).readLock(); + lock.lock(); + return new LockUnlocker(lock); } /** * Acquire write lock for provided key. */ - public CloseableLock acquireWriteLock(String key) { - return new WriteLock(key); + public Unlocker writeLock(String key) { + Lock lock = striped.get(key).writeLock(); + lock.lock(); + return new LockUnlocker(lock); } /** * Acquire write lock for all possible keys. */ - public CloseableLock acquireWriteAllLock() { - return new WriteAllLock(); - } - - /** - * Represents read lock for the provided key. - * Can be used as {@link AutoCloseable} to release lock. - */ - private class ReadLock implements CloseableLock { - private String key; - - private ReadLock(String key) { - this.key = key; - striped.get(key).readLock().lock(); + public Unlocker writeAllLock() { + Lock[] locks = getAllWriteLocks(); + for (Lock lock : locks) { + lock.lock(); } + return new LocksUnlocker(locks); + } - @Override - public void close() { - striped.get(key).readLock().unlock(); + private Lock[] getAllWriteLocks() { + Lock[] locks = new Lock[striped.size()]; + for (int i = 0; i < striped.size(); i++) { + locks[i] = striped.getAt(i).writeLock(); } + return locks; } - /** - * Represents write lock for the provided key. - * Can be used as {@link AutoCloseable} to release lock. - */ - private class WriteLock implements CloseableLock { - private String key; + private static class LockUnlocker implements Unlocker { - private WriteLock(String key) { - this.key = key; - striped.get(key).writeLock().lock(); + private final Lock lock; + + private LockUnlocker(Lock lock) { + this.lock = lock; } @Override - public void close() { - striped.get(key).writeLock().unlock(); + public void unlock() { + lock.unlock(); } } - /** - * Represents write lock for all possible keys. - * Can be used as {@link AutoCloseable} to release locks. - */ - private class WriteAllLock implements CloseableLock { - private WriteAllLock() { - for (int i = 0; i < striped.size(); i++) { - striped.getAt(i).writeLock().lock(); - } + private static class LocksUnlocker implements Unlocker { + + private final Lock[] locks; + + private LocksUnlocker(Lock[] locks) { + this.locks = locks; } @Override - public void close() { - for (int i = 0; i < striped.size(); i++) { - striped.getAt(i).writeLock().unlock(); + public void unlock() { + for (Lock lock : locks) { + lock.unlock(); } } } diff --git a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/CloseableLock.java b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/Unlocker.java similarity index 53% rename from core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/CloseableLock.java rename to core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/Unlocker.java index 18474ba5677..163d37dc946 100644 --- a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/CloseableLock.java +++ b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/concurrent/Unlocker.java @@ -11,20 +11,28 @@ package org.eclipse.che.commons.lang.concurrent; /** - * Lock that is designed to use in try-with-resources statement. + * An interface that allows implementations to enclose + * locked instance and unlock it later by calling {@link #unlock()}. * - *

Implementers should lock on instance creation - * and unlock when {@link CloseableLock#close()} method invokes. + *

This is designed to be used in try-with-resources statement. + * + *

The example: + * 

+ *     try (@SuppressWarnings("unused") Unlocker u = customLocks.lock("key")) {
+ *         // do something in lock
+ *     }
+ *
* * @author Sergii Leschenko + * @author Yevhenii Voevodin */ -public interface CloseableLock extends AutoCloseable { +public interface Unlocker extends AutoCloseable { + /** - * Unlocks this lock. - * - * This method is invoked automatically on objects managed by the - * {@code try}-with-resources statement. + * Unlocks the corresponding lock in implementation specific manner. */ + void unlock(); + @Override - void close(); + default void close() { unlock(); } } diff --git a/plugins/plugin-docker/che-plugin-docker-client/src/main/java/org/eclipse/che/plugin/docker/client/DockerConnector.java b/plugins/plugin-docker/che-plugin-docker-client/src/main/java/org/eclipse/che/plugin/docker/client/DockerConnector.java index a1377cc3d7b..a4592f26819 100644 --- a/plugins/plugin-docker/che-plugin-docker-client/src/main/java/org/eclipse/che/plugin/docker/client/DockerConnector.java +++ b/plugins/plugin-docker/che-plugin-docker-client/src/main/java/org/eclipse/che/plugin/docker/client/DockerConnector.java @@ -826,6 +826,7 @@ private String buildImage(final DockerConnection dockerConnection, throw new DockerException(e.getCause().getLocalizedMessage(), 500); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new DockerException("Docker image build was interrupted", 500); } } @@ -952,6 +953,7 @@ public String push(final PushParams params, final ProgressMonitor progressMonito // unwrap exception thrown by task with .getCause() throw new DockerException("Docker image pushing failed. Cause: " + e.getCause().getLocalizedMessage(), 500); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new DockerException("Docker image pushing was interrupted", 500); } } @@ -1048,6 +1050,7 @@ protected void pull(final PullParams params, // unwrap exception thrown by task with .getCause() throw new DockerException(e.getCause().getLocalizedMessage(), 500); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new DockerException("Docker image pulling was interrupted", 500); } } diff --git a/plugins/plugin-docker/che-plugin-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/MachineProviderImpl.java b/plugins/plugin-docker/che-plugin-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/MachineProviderImpl.java index f8b35131e34..ec70fac0e45 100644 --- a/plugins/plugin-docker/che-plugin-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/MachineProviderImpl.java +++ b/plugins/plugin-docker/che-plugin-docker-machine/src/main/java/org/eclipse/che/plugin/docker/machine/MachineProviderImpl.java @@ -471,8 +471,7 @@ protected void pullImage(CheServiceImpl service, docker.removeImage(RemoveImageParams.create(fullNameOfPulledImage).withForce(false)); } } catch (IOException e) { - LOG.error(e.getLocalizedMessage(), e); - throw new MachineException("Can't create machine from image. Cause: " + e.getLocalizedMessage()); + throw new MachineException("Can't create machine from image. Cause: " + e.getLocalizedMessage(), e); } } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentEngine.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentEngine.java index 66c4fb8730e..65092b78bd1 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentEngine.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/CheEnvironmentEngine.java @@ -38,6 +38,7 @@ import org.eclipse.che.api.core.util.lineconsumer.ConcurrentFileLineConsumer; import org.eclipse.che.api.environment.server.exception.EnvironmentException; import org.eclipse.che.api.environment.server.exception.EnvironmentNotRunningException; +import org.eclipse.che.api.environment.server.exception.EnvironmentStartInterruptedException; import org.eclipse.che.api.environment.server.model.CheServiceBuildContextImpl; import org.eclipse.che.api.environment.server.model.CheServiceImpl; import org.eclipse.che.api.environment.server.model.CheServicesEnvironmentImpl; @@ -57,13 +58,13 @@ import org.eclipse.che.api.machine.server.spi.SnapshotDao; import org.eclipse.che.api.machine.server.util.RecipeDownloader; import org.eclipse.che.api.machine.shared.dto.event.MachineStatusEvent; +import org.eclipse.che.commons.lang.concurrent.Unlocker; +import org.eclipse.che.commons.lang.concurrent.StripedLocks; import org.eclipse.che.api.workspace.server.model.impl.ExtendedMachineImpl; import org.eclipse.che.commons.env.EnvironmentContext; import org.eclipse.che.commons.lang.IoUtil; import org.eclipse.che.commons.lang.NameGenerator; import org.eclipse.che.commons.lang.Size; -import org.eclipse.che.commons.lang.concurrent.CloseableLock; -import org.eclipse.che.commons.lang.concurrent.StripedLocks; import org.slf4j.Logger; import javax.annotation.PostConstruct; @@ -100,7 +101,8 @@ @Singleton public class CheEnvironmentEngine { - private static final Logger LOG = getLogger(CheEnvironmentEngine.class); + private static final NoOpStartedHandler NO_OP_HANDLER = new NoOpStartedHandler(); + private static final Logger LOG = getLogger(CheEnvironmentEngine.class); private final Map environments; private final StripedLocks stripedLocks; @@ -167,7 +169,7 @@ public CheEnvironmentEngine(SnapshotDao snapshotDao, */ public List getMachines(String workspaceId) throws EnvironmentNotRunningException { EnvironmentHolder environment; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { environment = environments.get(workspaceId); if (environment == null) { throw new EnvironmentNotRunningException("Environment with ID '" + workspaceId + "' is not found"); @@ -191,7 +193,7 @@ public List getMachines(String workspaceId) throws EnvironmentNotRunni */ public Instance getMachine(String workspaceId, String machineId) throws NotFoundException { EnvironmentHolder environment; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { environment = environments.get(workspaceId); } if (environment == null) { @@ -207,8 +209,8 @@ public Instance getMachine(String workspaceId, String machineId) throws NotFound /** * Starts provided environment. - *

- * Environment starts if and only all machines in environment definition start successfully.
+ * + *

Environment starts if and only all machines in environment definition start successfully.
* Otherwise exception is thrown by this method.
* It is not defined whether environment start fails right after first failure or in the end of the process.
* Starting order of machines is not guarantied. Machines can start sequentially or in parallel. @@ -223,6 +225,8 @@ public Instance getMachine(String workspaceId, String machineId) throws NotFound * whether machines from environment should be recovered or not * @param messageConsumer * consumer of log messages from machines in the environment + * @param startedHandler + * handler for started machines * @return list of running machines of this environment * @throws ServerException * if other error occurs @@ -231,9 +235,10 @@ public List start(String workspaceId, String envName, Environment env, boolean recover, - MessageConsumer messageConsumer) throws ServerException, - ConflictException, - EnvironmentException { + MessageConsumer messageConsumer, + MachineStartedHandler startedHandler) throws ServerException, + EnvironmentException, + ConflictException { // TODO move to machines provider // add random chars to ensure that old environments that weren't removed by some reason won't prevent start String networkId = NameGenerator.generate(workspaceId + "_", 16); @@ -256,9 +261,10 @@ public List start(String workspaceId, workspaceId, devMachineName, networkId, - recover); + recover, + startedHandler); - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { EnvironmentHolder environmentHolder = environments.get(workspaceId); // possible only if environment was stopped during its start if (environmentHolder == null) { @@ -270,6 +276,21 @@ public List start(String workspaceId, } } + /** + * Starts workspace environment. + * + * @see #start(String, String, Environment, boolean, MessageConsumer, MachineStartedHandler) + */ + public List start(String workspaceId, + String envName, + Environment env, + boolean recover, + MessageConsumer messageConsumer) throws ServerException, + ConflictException, + EnvironmentException { + return start(workspaceId, envName, env, recover, messageConsumer, NO_OP_HANDLER); + } + /** * Stops running environment of specified workspace. * @@ -284,7 +305,7 @@ public void stop(String workspaceId) throws EnvironmentNotRunningException, ServerException { List machinesCopy = null; EnvironmentHolder environmentHolder; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { environmentHolder = environments.get(workspaceId); if (environmentHolder == null || environmentHolder.status != EnvStatus.RUNNING) { throw new EnvironmentNotRunningException( @@ -302,7 +323,7 @@ public void stop(String workspaceId) throws EnvironmentNotRunningException, destroyEnvironment(environmentHolder.networkId, machinesCopy); } - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { environments.remove(workspaceId); } } @@ -333,7 +354,7 @@ public Instance startMachine(String workspaceId, MachineConfig machineConfigCopy = new MachineConfigImpl(machineConfig); EnvironmentHolder environmentHolder; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { environmentHolder = environments.get(workspaceId); if (environmentHolder == null || environmentHolder.status != EnvStatus.RUNNING) { throw new EnvironmentNotRunningException(format("Environment '%s' is not running", workspaceId)); @@ -428,7 +449,7 @@ public void stopMachine(String workspaceId, String machineId) throws NotFoundExc ServerException, ConflictException { Instance targetMachine = null; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder == null || environmentHolder.status != EnvStatus.RUNNING) { throw new EnvironmentNotRunningException(format("Environment '%s' is not running", workspaceId)); @@ -475,7 +496,7 @@ public SnapshotImpl saveSnapshot(String workspaceId, EnvironmentHolder environmentHolder; SnapshotImpl snapshot = null; Instance instance = null; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { environmentHolder = environments.get(workspaceId); if (environmentHolder == null || environmentHolder.status != EnvStatus.RUNNING) { throw new EnvironmentNotRunningException(format("Environment '%s' is not running", workspaceId)); @@ -560,7 +581,7 @@ private void initializeEnvironment(String namespace, envName, networkId); - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { if (environments.putIfAbsent(workspaceId, environmentHolder) != null) { throw new ConflictException(format("Environment of workspace '%s' already exists", workspaceId)); } @@ -698,7 +719,8 @@ private void startEnvironmentQueue(String namespace, String workspaceId, String devMachineName, String networkId, - boolean recover) + boolean recover, + MachineStartedHandler startedHandler) throws ServerException, EnvironmentException { // Starting all machines in environment one by one by getting configs @@ -706,7 +728,7 @@ private void startEnvironmentQueue(String namespace, // Config will be null only if there are no machines left in the queue String envName; MessageConsumer envLogger; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder == null) { throw new ServerException("Environment start is interrupted."); @@ -727,7 +749,7 @@ private void startEnvironmentQueue(String namespace, String creator = EnvironmentContext.getCurrent().getSubject().getUserId(); CheServiceImpl service; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder == null) { throw new ServerException("Environment start is interrupted."); @@ -774,10 +796,15 @@ private void startEnvironmentQueue(String namespace, .setOwner(creator) .build(); + checkInterruption(workspaceId, envName); Instance instance = startInstance(recover, envLogger, machine, machineStarter); + checkInterruption(workspaceId, envName); + + startedHandler.started(instance); + checkInterruption(workspaceId, envName); // Machine destroying is an expensive operation which must be // performed outside of the lock, this section checks if @@ -785,7 +812,7 @@ private void startEnvironmentQueue(String namespace, // polled flag to true if the environment wasn't stopped. // Also polls the proceeded machine configuration from the queue boolean queuePolled = false; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { ensurePreDestroyIsNotExecuted(); EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder != null) { @@ -828,9 +855,10 @@ private void startEnvironmentQueue(String namespace, machineName = queuePeekOrFail(workspaceId); } - } catch (RuntimeException | ServerException e) { + } catch (RuntimeException | ServerException | EnvironmentStartInterruptedException e) { + boolean interrupted = Thread.interrupted(); EnvironmentHolder env; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { env = environments.remove(workspaceId); } @@ -839,7 +867,23 @@ private void startEnvironmentQueue(String namespace, } catch (Exception remEx) { LOG.error(remEx.getLocalizedMessage(), remEx); } - throw new ServerException(e.getLocalizedMessage(), e); + + if (interrupted) { + throw new EnvironmentStartInterruptedException(workspaceId, envName); + } + try { + throw e; + } catch (ServerException | EnvironmentStartInterruptedException rethrow) { + throw rethrow; + } catch (Exception wrap) { + throw new ServerException(wrap.getMessage(), wrap); + } + } + } + + private void checkInterruption(String workspaceId, String envName) throws EnvironmentStartInterruptedException { + if (Thread.interrupted()) { + throw new EnvironmentStartInterruptedException(workspaceId, envName); } } @@ -912,6 +956,8 @@ private Instance startInstance(boolean recover, return instance; } catch (ApiException | RuntimeException e) { + boolean interrupted = Thread.interrupted(); + removeMachine(machine.getWorkspaceId(), machine.getId()); if (instance != null) { @@ -942,6 +988,9 @@ private Instance startInstance(boolean recover, .withMachineId(machine.getId()) .withWorkspaceId(machine.getWorkspaceId())); + if (interrupted) { + Thread.currentThread().interrupt(); + } throw new ServerException(e.getLocalizedMessage(), e); } } @@ -998,7 +1047,7 @@ private Machine normalizeMachineSource(MachineImpl machine, MachineSource machin private void addMachine(MachineImpl machine) throws ServerException { Instance instance = new NoOpMachineInstance(machine); - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(machine.getWorkspaceId())) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(machine.getWorkspaceId())) { ensurePreDestroyIsNotExecuted(); EnvironmentHolder environmentHolder = environments.get(machine.getWorkspaceId()); if (environmentHolder != null && environmentHolder.status != EnvStatus.STOPPING) { @@ -1017,7 +1066,7 @@ private int bytesToMB(long bytes) { private void removeMachine(String workspaceId, String machineId) { - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) { EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder != null) { for (Instance machine : environmentHolder.machines) { @@ -1031,7 +1080,7 @@ private void removeMachine(String workspaceId, } private void replaceMachine(Instance machine) throws ServerException { - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireWriteLock(machine.getWorkspaceId())) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(machine.getWorkspaceId())) { ensurePreDestroyIsNotExecuted(); EnvironmentHolder environmentHolder = environments.get(machine.getWorkspaceId()); if (environmentHolder != null) { @@ -1071,7 +1120,7 @@ private void replaceMachine(Instance machine) throws ServerException { * if pre destroy has been invoked before peek config retrieved */ private String queuePeekOrFail(String workspaceId) throws ServerException { - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) { ensurePreDestroyIsNotExecuted(); EnvironmentHolder environmentHolder = environments.get(workspaceId); if (environmentHolder == null || environmentHolder.startQueue == null) { @@ -1277,7 +1326,7 @@ private class MachineCleaner implements EventSubscriber { public void onEvent(InstanceStateEvent event) { if ((event.getType() == OOM) || (event.getType() == DIE)) { EnvironmentHolder environmentHolder; - try (@SuppressWarnings("unused") CloseableLock lock = stripedLocks.acquireReadLock("workspaceId")) { + try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock("workspaceId")) { environmentHolder = environments.get(event.getWorkspaceId()); } if (environmentHolder != null) { @@ -1311,4 +1360,9 @@ public void onEvent(InstanceStateEvent event) { } } } + + private static class NoOpStartedHandler implements MachineStartedHandler { + @Override + public void started(Instance machine) throws ServerException {} + } } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/MachineStartedHandler.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/MachineStartedHandler.java new file mode 100644 index 00000000000..d1094ea0040 --- /dev/null +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/MachineStartedHandler.java @@ -0,0 +1,27 @@ +/******************************************************************************* + * Copyright (c) 2012-2017 Codenvy, S.A. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Codenvy, S.A. - initial API and implementation + *******************************************************************************/ +package org.eclipse.che.api.environment.server; + +import org.eclipse.che.api.core.ServerException; +import org.eclipse.che.api.environment.server.exception.EnvironmentException; +import org.eclipse.che.api.machine.server.spi.Instance; + +/** + * Used in couple with {@link CheEnvironmentEngine#start} method to + * allow sequential handling and interruption of the start process. + * + *

This interface is a part of a contract for {@link CheEnvironmentEngine}. + * + * @author Yevhenii Voevodin + */ +public interface MachineStartedHandler { + void started(Instance machine) throws EnvironmentException, ServerException; +} diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/exception/EnvironmentStartInterruptedException.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/exception/EnvironmentStartInterruptedException.java new file mode 100644 index 00000000000..21629b71e6d --- /dev/null +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/environment/server/exception/EnvironmentStartInterruptedException.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2012-2017 Codenvy, S.A. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Codenvy, S.A. - initial API and implementation + *******************************************************************************/ +package org.eclipse.che.api.environment.server.exception; + +/** + * Thrown when environment start is interrupted. + * + * @author Yevhenii Voevodin + */ +public class EnvironmentStartInterruptedException extends EnvironmentException { + public EnvironmentStartInterruptedException(String workspaceId, String envName) { + super(String.format("Start of environment '%s' in workspace '%s' is interrupted", + envName, + workspaceId)); + } +} diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceManager.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceManager.java index 56bcecc576d..585416bdb49 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceManager.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceManager.java @@ -29,7 +29,6 @@ import org.eclipse.che.api.machine.server.model.impl.SnapshotImpl; import org.eclipse.che.api.machine.server.spi.Instance; import org.eclipse.che.api.machine.server.spi.SnapshotDao; -import org.eclipse.che.api.workspace.server.WorkspaceRuntimes.RuntimeDescriptor; import org.eclipse.che.api.workspace.server.event.WorkspaceCreatedEvent; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceConfigImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceImpl; @@ -46,8 +45,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Throwables.getCausalChain; @@ -130,10 +127,12 @@ public WorkspaceImpl createWorkspace(WorkspaceConfig config, NotFoundException { requireNonNull(config, "Required non-null config"); requireNonNull(namespace, "Required non-null namespace"); - return normalizeState(doCreateWorkspace(config, - accountManager.getByName(namespace), - emptyMap(), - false)); + WorkspaceImpl workspace = doCreateWorkspace(config, + accountManager.getByName(namespace), + emptyMap(), + false); + workspace.setStatus(WorkspaceStatus.STOPPED); + return workspace; } /** @@ -164,10 +163,12 @@ public WorkspaceImpl createWorkspace(WorkspaceConfig config, requireNonNull(config, "Required non-null config"); requireNonNull(namespace, "Required non-null namespace"); requireNonNull(attributes, "Required non-null attributes"); - return normalizeState(doCreateWorkspace(config, - accountManager.getByName(namespace), - attributes, - false)); + WorkspaceImpl workspace = doCreateWorkspace(config, + accountManager.getByName(namespace), + emptyMap(), + false); + workspace.setStatus(WorkspaceStatus.STOPPED); + return workspace; } /** @@ -193,7 +194,9 @@ public WorkspaceImpl createWorkspace(WorkspaceConfig config, */ public WorkspaceImpl getWorkspace(String key) throws NotFoundException, ServerException { requireNonNull(key, "Required non-null workspace key"); - return normalizeState(getByKey(key)); + WorkspaceImpl workspace = getByKey(key); + runtimes.injectRuntime(workspace); + return workspace; } /** @@ -215,7 +218,9 @@ public WorkspaceImpl getWorkspace(String key) throws NotFoundException, ServerEx public WorkspaceImpl getWorkspace(String name, String namespace) throws NotFoundException, ServerException { requireNonNull(name, "Required non-null workspace name"); requireNonNull(namespace, "Required non-null workspace owner"); - return normalizeState(workspaceDao.get(name, namespace)); + WorkspaceImpl workspace = workspaceDao.get(name, namespace); + runtimes.injectRuntime(workspace); + return workspace; } /** @@ -256,8 +261,10 @@ public List getWorkspaces(String user) throws ServerException { public List getWorkspaces(String user, boolean includeRuntimes) throws ServerException { requireNonNull(user, "Required non-null user id"); final List workspaces = workspaceDao.getWorkspaces(user); - for (WorkspaceImpl workspace : workspaces) { - normalizeState(workspace, includeRuntimes); + if (includeRuntimes) { + injectRuntimes(workspaces); + } else { + injectStatuses(workspaces); } return workspaces; } @@ -300,8 +307,10 @@ public List getByNamespace(String namespace) throws ServerExcepti public List getByNamespace(String namespace, boolean includeRuntimes) throws ServerException { requireNonNull(namespace, "Required non-null namespace"); final List workspaces = workspaceDao.getByNamespace(namespace); - for (WorkspaceImpl workspace : workspaces) { - normalizeState(workspace, includeRuntimes); + if (includeRuntimes) { + injectRuntimes(workspaces); + } else { + injectStatuses(workspaces); } return workspaces; } @@ -329,12 +338,14 @@ public WorkspaceImpl updateWorkspace(String id, Workspace update) throws Conflic NotFoundException { requireNonNull(id, "Required non-null workspace id"); requireNonNull(update, "Required non-null workspace update"); - final WorkspaceImpl workspace = workspaceDao.get(id); + WorkspaceImpl workspace = workspaceDao.get(id); workspace.setConfig(new WorkspaceConfigImpl(update.getConfig())); update.getAttributes().put(UPDATED_ATTRIBUTE_NAME, Long.toString(currentTimeMillis())); workspace.setAttributes(update.getAttributes()); workspace.setTemporary(update.isTemporary()); - return normalizeState(workspaceDao.update(workspace)); + WorkspaceImpl updated = workspaceDao.update(workspace); + runtimes.injectRuntime(updated); + return updated; } /** @@ -401,7 +412,8 @@ public WorkspaceImpl startWorkspace(String workspaceId, final String restoreAttr = workspace.getAttributes().get(AUTO_RESTORE_FROM_SNAPSHOT); final boolean autoRestore = restoreAttr == null ? defaultAutoRestore : parseBoolean(restoreAttr); startAsync(workspace, envName, firstNonNull(restore, autoRestore) && !getSnapshot(workspaceId).isEmpty()); - return normalizeState(workspace); + runtimes.injectRuntime(workspace); + return workspace; } /** @@ -431,7 +443,8 @@ public WorkspaceImpl startWorkspace(WorkspaceConfig config, emptyMap(), isTemporary); startAsync(workspace, workspace.getConfig().getDefaultEnv(), false); - return normalizeState(workspace); + runtimes.injectRuntime(workspace); + return workspace; } /** @@ -506,8 +519,8 @@ public void stopWorkspace(String workspaceId, @Nullable Boolean createSnapshot) NotFoundException, ServerException { requireNonNull(workspaceId, "Required non-null workspace id"); - final WorkspaceImpl workspace = normalizeState(workspaceDao.get(workspaceId)); - checkWorkspaceIsRunning(workspace, "stop"); + final WorkspaceImpl workspace = workspaceDao.get(workspaceId); + workspace.setStatus(runtimes.getStatus(workspaceId)); stopAsync(workspace, createSnapshot); } @@ -615,7 +628,8 @@ public void stopMachine(String workspaceId, ConflictException { requireNonNull(workspaceId, "Required non-null workspace id"); requireNonNull(machineId, "Required non-null machine id"); - final WorkspaceImpl workspace = normalizeState(workspaceDao.get(workspaceId)); + final WorkspaceImpl workspace = workspaceDao.get(workspaceId); + workspace.setStatus(runtimes.getStatus(workspaceId)); checkWorkspaceIsRunning(workspace, format("stop machine with ID '%s' of", machineId)); runtimes.stopMachine(workspaceId, machineId); } @@ -638,7 +652,7 @@ public Instance getMachineInstance(String workspaceId, ServerException { requireNonNull(workspaceId, "Required non-null workspace id"); requireNonNull(machineId, "Required non-null machine id"); - normalizeState(workspaceDao.get(workspaceId)); + workspaceDao.get(workspaceId); return runtimes.getMachine(workspaceId, machineId); } @@ -658,33 +672,46 @@ private void startAsync(WorkspaceImpl workspace, workspaceDao.update(workspace); final String env = firstNonNull(envName, workspace.getConfig().getDefaultEnv()); - // barrier, safely doesn't allow to start the workspace twice - final Future descriptor = runtimes.startAsync(workspace, env, recover); - - sharedPool.execute(() -> { - try { - descriptor.get(); - LOG.info("Workspace '{}:{}' with id '{}' started by user '{}'", - workspace.getNamespace(), - workspace.getConfig().getName(), - workspace.getId(), - sessionUserNameOr("undefined")); - } catch (Exception ex) { - if (workspace.isTemporary()) { - removeWorkspaceQuietly(workspace); - } - for (Throwable cause : getCausalChain(ex)) { - if (cause instanceof SourceNotFoundException) { - return; + runtimes.startAsync(workspace, env, recover) + .whenComplete((runtime, ex) -> { + if (ex == null) { + LOG.info("Workspace '{}:{}' with id '{}' started by user '{}'", + workspace.getNamespace(), + workspace.getConfig().getName(), + workspace.getId(), + sessionUserNameOr("undefined")); + } else { + if (workspace.isTemporary()) { + removeWorkspaceQuietly(workspace); + } + for (Throwable cause : getCausalChain(ex)) { + if (cause instanceof SourceNotFoundException) { + return; + } + } + try { + throw ex; + } catch (EnvironmentException envEx) { + // it's okay, e.g. recipe is invalid | start interrupted + LOG.info("Workspace '{}:{}' can't be started because: {}", + workspace.getNamespace(), + workspace.getConfig().getName(), + envEx.getMessage()); + } catch (Throwable thr) { + LOG.error(thr.getMessage(), thr); + } } - } - LOG.error(ex.getLocalizedMessage(), ex); - } - }); + }); } private void stopAsync(WorkspaceImpl workspace, @Nullable Boolean createSnapshot) throws ConflictException { - checkWorkspaceIsRunning(workspace, "stop"); + if (workspace.getStatus() != WorkspaceStatus.RUNNING && workspace.getStatus() != WorkspaceStatus.STARTING) { + throw new ConflictException(format("Could not stop the workspace '%s:%s' because its status is '%s'. " + + "Workspace must be either 'STARTING' or 'RUNNING'", + workspace.getNamespace(), + workspace.getConfig().getName(), + workspace.getStatus())); + } sharedPool.execute(() -> { final String stoppedBy = sessionUserNameOr(workspace.getAttributes().get(WORKSPACE_STOPPED_BY)); @@ -695,7 +722,7 @@ private void stopAsync(WorkspaceImpl workspace, @Nullable Boolean createSnapshot firstNonNull(stoppedBy, "undefined")); final boolean snapshotBeforeStop; - if (workspace.isTemporary()) { + if (workspace.isTemporary() || workspace.getStatus() == WorkspaceStatus.STARTING) { snapshotBeforeStop = false; } else if (createSnapshot != null) { snapshotBeforeStop = createSnapshot; @@ -728,7 +755,7 @@ private void stopAsync(WorkspaceImpl workspace, @Nullable Boolean createSnapshot workspace.getConfig().getName(), workspace.getId(), firstNonNull(stoppedBy, "undefined")); - } catch (RuntimeException | ConflictException | NotFoundException | ServerException ex) { + } catch (Exception ex) { LOG.error(ex.getLocalizedMessage(), ex); } finally { if (workspace.isTemporary()) { @@ -778,33 +805,6 @@ private String sessionUserNameOr(String nameIfNoUser) { return nameIfNoUser; } - private WorkspaceImpl normalizeState(WorkspaceImpl workspace) throws ServerException { - return normalizeState(workspace, true); - } - - private WorkspaceImpl normalizeState(WorkspaceImpl workspace, boolean includeRuntimes) throws ServerException { - if (includeRuntimes) { - try { - return normalizeState(workspace, runtimes.get(workspace.getId())); - } catch (NotFoundException e) { - return normalizeState(workspace, null); - } - } else { - workspace.setStatus(runtimes.getStatus(workspace.getId())); - return workspace; - } - } - - private WorkspaceImpl normalizeState(WorkspaceImpl workspace, RuntimeDescriptor descriptor) { - if (descriptor != null) { - workspace.setStatus(descriptor.getRuntimeStatus()); - workspace.setRuntime(descriptor.getRuntime()); - } else { - workspace.setStatus(WorkspaceStatus.STOPPED); - } - return workspace; - } - private WorkspaceImpl doCreateWorkspace(WorkspaceConfig config, Account account, Map attributes, @@ -843,4 +843,17 @@ private WorkspaceImpl getByKey(String key) throws NotFoundException, ServerExcep final String namespace = nsPart.isEmpty() ? sessionUser().getUserName() : nsPart; return workspaceDao.get(wsName, namespace); } + + + private void injectRuntimes(List workspaces) { + for (WorkspaceImpl workspace : workspaces) { + runtimes.injectRuntime(workspace); + } + } + + private void injectStatuses(List workspaces) { + for (WorkspaceImpl workspace : workspaces) { + workspace.setStatus(runtimes.getStatus(workspace.getId())); + } + } } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java index 9df69f7d712..339ce5cdb24 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimes.java @@ -20,25 +20,21 @@ import org.eclipse.che.api.agent.server.launcher.AgentLauncherFactory; import org.eclipse.che.api.agent.shared.model.Agent; import org.eclipse.che.api.agent.shared.model.AgentKey; -import org.eclipse.che.api.core.ApiException; import org.eclipse.che.api.core.ConflictException; import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.core.model.machine.MachineConfig; -import org.eclipse.che.api.core.model.machine.MachineLogMessage; -import org.eclipse.che.api.core.model.machine.MachineStatus; import org.eclipse.che.api.core.model.workspace.Environment; import org.eclipse.che.api.core.model.workspace.ExtendedMachine; import org.eclipse.che.api.core.model.workspace.Workspace; -import org.eclipse.che.api.core.model.workspace.WorkspaceRuntime; import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; import org.eclipse.che.api.core.notification.EventService; -import org.eclipse.che.api.core.util.AbstractMessageConsumer; -import org.eclipse.che.api.core.util.MessageConsumer; import org.eclipse.che.api.core.util.WebsocketMessageConsumer; import org.eclipse.che.api.environment.server.CheEnvironmentEngine; +import org.eclipse.che.api.environment.server.MachineStartedHandler; import org.eclipse.che.api.environment.server.exception.EnvironmentException; import org.eclipse.che.api.environment.server.exception.EnvironmentNotRunningException; +import org.eclipse.che.api.environment.server.exception.EnvironmentStartInterruptedException; import org.eclipse.che.api.machine.server.exception.MachineException; import org.eclipse.che.api.machine.server.exception.SnapshotException; import org.eclipse.che.api.machine.server.model.impl.MachineConfigImpl; @@ -47,39 +43,44 @@ import org.eclipse.che.api.machine.server.spi.Instance; import org.eclipse.che.api.machine.server.spi.SnapshotDao; import org.eclipse.che.api.workspace.server.model.impl.EnvironmentImpl; -import org.eclipse.che.api.workspace.server.model.impl.ExtendedMachineImpl; +import org.eclipse.che.api.workspace.server.model.impl.WorkspaceImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceRuntimeImpl; import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent.EventType; -import org.eclipse.che.commons.lang.concurrent.CloseableLock; import org.eclipse.che.commons.lang.concurrent.StripedLocks; +import org.eclipse.che.commons.lang.concurrent.Unlocker; import org.eclipse.che.dto.server.DtoFactory; import org.slf4j.Logger; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import static java.lang.String.format; import static java.util.Comparator.comparing; +import static java.util.Objects.requireNonNull; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.RUNNING; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.SNAPSHOTTING; +import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STARTING; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STOPPING; import static org.eclipse.che.api.machine.shared.Constants.ENVIRONMENT_OUTPUT_CHANNEL_TEMPLATE; import static org.slf4j.LoggerFactory.getLogger; @@ -89,16 +90,14 @@ * *

This component implements {@link WorkspaceStatus} contract. * - *

All the operations performed by this component are synchronous. - * *

The implementation is thread-safe and guarded by * eagerly initialized readwrite locks produced by {@link StripedLocks}. * The component doesn't expose any api for client-side locking. * All the instances produced by this component are copies of the real data. * *

The component doesn't check if the incoming objects are in application-valid state. - * Which means that it is expected that if {@link #start(Workspace, String, boolean)} method is called - * then {@code WorkspaceImpl} argument is a application-valid object which contains + * Which means that it is expected that if {@link #startAsync(Workspace, String, boolean)} method is called + * then {@code Workspace} argument is a application-valid object which contains * all the required data for performing start. * * @author Yevhenii Voevodin @@ -109,15 +108,15 @@ public class WorkspaceRuntimes { private static final Logger LOG = getLogger(WorkspaceRuntimes.class); - private final ConcurrentMap workspaces; - private final EventService eventsService; - private final StripedLocks locks; - private final CheEnvironmentEngine envEngine; - private final AgentSorter agentSorter; - private final AgentLauncherFactory launcherFactory; - private final AgentRegistry agentRegistry; - private final SnapshotDao snapshotDao; - private final WorkspaceSharedPool sharedPool; + private final ConcurrentMap states; + private final EventService eventsService; + private final StripedLocks locks; + private final CheEnvironmentEngine envEngine; + private final AgentSorter agentSorter; + private final AgentLauncherFactory launcherFactory; + private final AgentRegistry agentRegistry; + private final SnapshotDao snapshotDao; + private final WorkspaceSharedPool sharedPool; private volatile boolean isPreDestroyInvoked; @@ -129,27 +128,119 @@ public WorkspaceRuntimes(EventService eventsService, AgentRegistry agentRegistry, SnapshotDao snapshotDao, WorkspaceSharedPool sharedPool) { + this(eventsService, + envEngine, + agentSorter, + launcherFactory, + agentRegistry, + snapshotDao, + sharedPool, + new ConcurrentHashMap<>()); + } + + public WorkspaceRuntimes(EventService eventsService, + CheEnvironmentEngine envEngine, + AgentSorter agentSorter, + AgentLauncherFactory launcherFactory, + AgentRegistry agentRegistry, + SnapshotDao snapshotDao, + WorkspaceSharedPool sharedPool, + ConcurrentMap states) { this.eventsService = eventsService; this.envEngine = envEngine; this.agentSorter = agentSorter; this.launcherFactory = launcherFactory; this.agentRegistry = agentRegistry; this.snapshotDao = snapshotDao; - this.workspaces = new ConcurrentHashMap<>(); // 16 - experimental value for stripes count, it comes from default hash map size this.locks = new StripedLocks(16); this.sharedPool = sharedPool; + this.states = states; } /** - * Returns the runtime descriptor describing currently starting/running/stopping - * workspace runtime. + * Asynchronously starts the environment of the workspace. + * Before executing start task checks whether all conditions + * are met and throws appropriate exceptions if not, so + * there is no way to start the same workspace twice. + * + *

Note that cancellation of resulting future won't + * interrupt workspace start, call {@link #stop(String)} directly instead. + * + *

If starting process is interrupted let's say within call + * to {@link #stop(String)} method, resulting future will + * be exceptionally completed(eventually) with an instance of + * {@link EnvironmentStartInterruptedException}. Note that clients + * don't have to cleanup runtime resources, the component + * will do necessary cleanup when interrupted. + * + *

Implementation notes: + * if thread which executes the task is interrupted, then the + * task is also eventually(depends on the environment engine implementation) + * interrupted as if {@link #stop(String)} is called directly. + * That helps to shutdown gracefully when thread pool is asked + * to {@link ExecutorService#shutdownNow()} and also reduces + * shutdown time when there are starting workspaces. * - *

Note that the {@link RuntimeDescriptor#getRuntime()} method - * returns a copy of a real {@code WorkspaceRuntime} object, - * which means that any runtime copy modifications won't affect the - * real object and also it means that copy won't be affected with modifications applied - * to the real runtime workspace object state. + * @param workspace + * workspace containing target environment + * @param envName + * the name of the environment to start + * @param recover + * whether to recover from the snapshot + * @return completable future describing the instance of running environment + * @throws ConflictException + * when the workspace is already started + * @throws IllegalArgumentException + * when the workspace doesn't contain the environment + * @throws NullPointerException + * when either {@code workspace} or {@code envName} is null + */ + public CompletableFuture startAsync(Workspace workspace, + String envName, + boolean recover) throws ConflictException { + requireNonNull(workspace, "Non-null workspace required"); + requireNonNull(envName, "Non-null environment name required"); + EnvironmentImpl environment = copyEnv(workspace, envName); + String workspaceId = workspace.getId(); + CompletableFuture cmpFuture; + StartTask startTask; + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { + ensurePreDestroyIsNotExecuted(); + RuntimeState state = states.get(workspaceId); + if (state != null) { + throw new ConflictException(format("Could not start workspace '%s' because its status is '%s'", + workspace.getConfig().getName(), + state.status)); + } + startTask = new StartTask(workspaceId, + envName, + environment, + recover, + cmpFuture = new CompletableFuture<>()); + states.put(workspaceId, new RuntimeState(WorkspaceStatus.STARTING, + envName, + startTask, + sharedPool.submit(startTask))); + } + + // publish event synchronously as the task may not be executed by + // executors service(due to legal cancellation), clients still have + // to receive STOPPED -> STARTING event + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withStatus(WorkspaceStatus.STARTING) + .withEventType(EventType.STARTING) + .withPrevStatus(WorkspaceStatus.STOPPED)); + + // so the start thread is free to go and start the environment + startTask.unlockStart(); + + return cmpFuture; + } + + /** + * Gets workspace runtime descriptor. * * @param workspaceId * the id of the workspace to get its runtime @@ -157,100 +248,74 @@ public WorkspaceRuntimes(EventService eventsService, * @throws NotFoundException * when workspace with given {@code workspaceId} is not found * @throws ServerException - * if environment is in illegal state + * if any error occurs while getting machines runtime information */ - public RuntimeDescriptor get(String workspaceId) throws NotFoundException, - ServerException { - WorkspaceState workspaceState; - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireReadLock(workspaceId)) { - workspaceState = workspaces.get(workspaceId); - } - if (workspaceState == null) { - throw new NotFoundException("Workspace with id '" + workspaceId + "' is not running."); - } - - RuntimeDescriptor runtimeDescriptor = new RuntimeDescriptor(workspaceState.status, - new WorkspaceRuntimeImpl(workspaceState.activeEnv, - null, - Collections.emptyList(), - null)); - List machines = envEngine.getMachines(workspaceId); - Optional devMachineOptional = machines.stream() - .filter(machine -> machine.getConfig().isDev()) - .findAny(); - if (devMachineOptional.isPresent()) { - String projectsRoot = devMachineOptional.get().getStatus() == MachineStatus.RUNNING ? - devMachineOptional.get().getRuntime().projectsRoot() : - null; - runtimeDescriptor.setRuntime(new WorkspaceRuntimeImpl(workspaceState.activeEnv, - projectsRoot, - machines, - devMachineOptional.get())); - } else if (workspaceState.status == WorkspaceStatus.RUNNING) { - // invalid state of environment is detected - String error = format("Dev machine is not found in active environment of workspace '%s'", - workspaceId); - throw new ServerException(error); - } - - return runtimeDescriptor; + public WorkspaceRuntimeImpl getRuntime(String workspaceId) throws NotFoundException, ServerException { + requireNonNull(workspaceId, "Required non-null workspace id"); + RuntimeState state; + try (@SuppressWarnings("unused") Unlocker u = locks.readLock(workspaceId)) { + state = new RuntimeState(getExistingState(workspaceId)); + } + return new WorkspaceRuntimeImpl(state.envName, envEngine.getMachines(workspaceId)); } /** - * Starts all machines from specified workspace environment, - * creates workspace runtime instance based on that environment. + * Return status of the workspace. * - *

During the start of the workspace its - * runtime is visible with {@link WorkspaceStatus#STARTING} status. + * @param workspaceId + * ID of requested workspace + * @return {@link WorkspaceStatus#STOPPED} if workspace is not running or, + * the status of workspace runtime otherwise + */ + public WorkspaceStatus getStatus(String workspaceId) { + requireNonNull(workspaceId, "Required non-null workspace id"); + try (@SuppressWarnings("unused") Unlocker u = locks.readLock(workspaceId)) { + RuntimeState state = states.get(workspaceId); + if (state == null) { + return WorkspaceStatus.STOPPED; + } + return state.status; + } + } + + /** + * Injects runtime information such as status and {@link WorkspaceRuntimeImpl} + * into the workspace object, if the workspace doesn't have runtime sets the + * status to {@link WorkspaceStatus#STOPPED}. * * @param workspace - * workspace which environment should be started - * @param envName - * the name of the environment to start - * @param recover - * whether machines should be recovered(true) or not(false) - * @return the workspace runtime instance with machines set. - * @throws ConflictException - * when workspace is already running - * @throws ConflictException - * when start is interrupted - * @throws NotFoundException - * when any not found exception occurs during environment start - * @throws ServerException - * when component {@link #isPreDestroyInvoked is stopped} - * @throws ServerException - * other error occurs during environment start - * @see CheEnvironmentEngine#start(String, String, Environment, boolean, MessageConsumer) - * @see WorkspaceStatus#STARTING - * @see WorkspaceStatus#RUNNING + * the workspace to inject runtime into */ - public RuntimeDescriptor start(Workspace workspace, - String envName, - boolean recover) throws ServerException, - ConflictException, - NotFoundException { - final EnvironmentImpl environment = copyEnv(workspace, envName); - final String workspaceId = workspace.getId(); - initState(workspaceId, workspace.getConfig().getName(), envName); - doStart(environment, workspaceId, envName, recover); - return get(workspaceId); + public void injectRuntime(WorkspaceImpl workspace) { + requireNonNull(workspace, "Required non-null workspace"); + RuntimeState state = null; + try (@SuppressWarnings("unused") Unlocker u = locks.readLock(workspace.getId())) { + if (states.containsKey(workspace.getId())) { + state = new RuntimeState(states.get(workspace.getId())); + } + } + if (state == null) { + workspace.setStatus(WorkspaceStatus.STOPPED); + } else { + workspace.setStatus(state.status); + try { + workspace.setRuntime(new WorkspaceRuntimeImpl(state.envName, envEngine.getMachines(workspace.getId()))); + } catch (Exception x) { + workspace.setRuntime(new WorkspaceRuntimeImpl(state.envName, Collections.emptyList())); + } + } } /** - * Starts the workspace like {@link #start(Workspace, String, boolean)} - * method does, but asynchronously. Nonetheless synchronously checks that workspace - * doesn't have runtime and makes it {@link WorkspaceStatus#STARTING}. + * Returns true if the status of the workspace is different + * from {@link WorkspaceStatus#STOPPED}. + * + * @param workspaceId + * workspace identifier to perform check + * @return true if workspace status is different from {@link WorkspaceStatus#STOPPED} */ - public Future startAsync(Workspace workspace, - String envName, - boolean recover) throws ConflictException, ServerException { - final EnvironmentImpl environment = copyEnv(workspace, envName); - final String workspaceId = workspace.getId(); - initState(workspaceId, workspace.getConfig().getName(), envName); - return sharedPool.submit(() -> { - doStart(environment, workspaceId, envName, recover); - return get(workspaceId); - }); + public boolean hasRuntime(String workspaceId) { + return states.containsKey(workspaceId); } /** @@ -271,86 +336,52 @@ public Future startAsync(Workspace workspace, * @see CheEnvironmentEngine#stop(String) * @see WorkspaceStatus#STOPPING */ - public void stop(String workspaceId) throws NotFoundException, ServerException, ConflictException { - // This check allows to exit with an appropriate exception before blocking on lock. - // The double check is required as it is still possible to get unlucky timing - // between locking and stopping workspace. - ensurePreDestroyIsNotExecuted(); - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireWriteLock(workspaceId)) { + public void stop(String workspaceId) throws NotFoundException, + ServerException, + ConflictException, + EnvironmentException { + requireNonNull(workspaceId, "Required not-null workspace id"); + RuntimeState prevState; + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { ensurePreDestroyIsNotExecuted(); - WorkspaceState workspaceState = workspaces.get(workspaceId); - if (workspaceState == null) { - throw new NotFoundException("Workspace with id '" + workspaceId + "' is not running."); - } - if (workspaceState.status != WorkspaceStatus.RUNNING) { - throw new ConflictException(format("Couldn't stop '%s' workspace because its status is '%s'. " + - "Workspace can be stopped only if it is 'RUNNING'", + RuntimeState state = getExistingState(workspaceId); + if (state.status != WorkspaceStatus.RUNNING && state.status != WorkspaceStatus.STARTING) { + throw new ConflictException(format("Couldn't stop the workspace '%s' because its status is '%s'. " + + "Workspace can be stopped only if it is 'RUNNING' or 'STARTING'", workspaceId, - workspaceState.status)); + state.status)); } - - workspaceState.status = WorkspaceStatus.STOPPING; + prevState = new RuntimeState(state); + state.status = WorkspaceStatus.STOPPING; } - eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspaceId) - .withPrevStatus(WorkspaceStatus.RUNNING) - .withStatus(WorkspaceStatus.STOPPING) - .withEventType(EventType.STOPPING)); - String error = null; - try { - envEngine.stop(workspaceId); - } catch (ServerException | RuntimeException e) { - error = e.getLocalizedMessage(); - } finally { - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireWriteLock(workspaceId)) { - workspaces.remove(workspaceId); - } + // workspace is running, stop normally + if (prevState.status == WorkspaceStatus.RUNNING) { + stopEnvironmentAndPublishEvents(workspaceId, WorkspaceStatus.RUNNING); + return; } - final WorkspaceStatusEvent event = DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspaceId) - .withPrevStatus(WorkspaceStatus.STOPPING); - if (error == null) { - event.setStatus(WorkspaceStatus.STOPPED); - event.setEventType(EventType.STOPPED); - } else { - event.setStatus(WorkspaceStatus.STOPPED); - event.setEventType(EventType.ERROR); - event.setError(error); + // interrupt workspace start thread + prevState.startFuture.cancel(true); + + // if task wasn't called by executor service, then + // no real machines were started but, the clients still + // have to be notified about the workspace shut down + StartTask startTask = prevState.startTask; + if (startTask.markAsUsed()) { + removeStateAndPublishStopEvents(workspaceId); + prevState.startTask.earlyComplete(); + return; } - eventsService.publish(event); - } - /** - * Returns true if workspace was started and its status is - * {@link WorkspaceStatus#RUNNING running}, {@link WorkspaceStatus#STARTING starting} - * or {@link WorkspaceStatus#STOPPING stopping} - otherwise returns false. - * - *

This method is less expensive alternative to {@link #get(String)} + {@code try catch}, see example: - * 

{@code
-     *
-     *     if (!runtimes.hasRuntime("workspace123")) {
-     *         doStuff("workspace123");
-     *     }
-     *
-     *     //vs
-     *
-     *     try {
-     *         runtimes.get("workspace123");
-     *     } catch (NotFoundException ex) {
-     *         doStuff("workspace123");
-     *     }
-     *
-     * }
- * - * @param workspaceId - * workspace identifier to perform check - * @return true if workspace is running, otherwise false - */ - public boolean hasRuntime(String workspaceId) { - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireReadLock(workspaceId)) { - return workspaces.containsKey(workspaceId); + // otherwise stop will be triggered by the start task, wait for it to finish + try { + startTask.await(); + } catch (EnvironmentStartInterruptedException ignored) { + // environment start successfully interrupted + } catch (InterruptedException x) { + Thread.currentThread().interrupt(); + throw new ServerException("Interrupted while waiting for start task cancellation", x); } } @@ -375,7 +406,7 @@ public Instance startMachine(String workspaceId, NotFoundException, EnvironmentException { - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = locks.readLock(workspaceId)) { getRunningState(workspaceId); } @@ -388,9 +419,9 @@ public Instance startMachine(String workspaceId, Instance instance = envEngine.startMachine(workspaceId, machineConfigCopy, agents); launchAgents(instance, agents); - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { ensurePreDestroyIsNotExecuted(); - WorkspaceState workspaceState = workspaces.get(workspaceId); + RuntimeState workspaceState = states.get(workspaceId); if (workspaceState == null || workspaceState.status != RUNNING) { try { envEngine.stopMachine(workspaceId, instance.getId()); @@ -423,7 +454,7 @@ public Instance startMachine(String workspaceId, public void snapshot(String workspaceId) throws NotFoundException, ConflictException, ServerException { - try (@SuppressWarnings("unused") CloseableLock l = locks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { getRunningState(workspaceId).status = SNAPSHOTTING; } snapshotAndUpdateStatus(workspaceId); @@ -437,7 +468,7 @@ public void snapshot(String workspaceId) throws NotFoundException, * @see #snapshot(String) */ public Future snapshotAsync(String workspaceId) throws NotFoundException, ConflictException { - try (@SuppressWarnings("unused") CloseableLock l = locks.acquireWriteLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { getRunningState(workspaceId).status = SNAPSHOTTING; } return sharedPool.submit(() -> { @@ -510,7 +541,7 @@ public void removeBinaries(Collection snapshots) { public void stopMachine(String workspaceId, String machineId) throws NotFoundException, ServerException, ConflictException { - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireReadLock(workspaceId)) { + try (@SuppressWarnings("unused") Unlocker u = locks.readLock(workspaceId)) { getRunningState(workspaceId); } envEngine.stopMachine(workspaceId, machineId); @@ -532,42 +563,19 @@ public Instance getMachine(String workspaceId, String machineId) throws NotFound } /** - * Returns all workspaces with statuses of its active environment. - */ - public Map getWorkspaces() { - return new HashMap<>(workspaces); - } - - /** - * Return status of the workspace. + * Gets the workspaces identifiers managed by this component. + * If an identifier is present in set then that workspace wasn't + * stopped at the moment of method execution. * - * @param workspaceId - * ID of requested workspace - * @return workspace status + * @return workspaces identifiers for those workspaces that are running(not stopped), + * or an empty set if there is no a single running workspace */ - public WorkspaceStatus getStatus(String workspaceId) { - try (@SuppressWarnings("unused") CloseableLock l = locks.acquireReadLock(workspaceId)) { - final WorkspaceState state = workspaces.get(workspaceId); - if (state == null) { - return WorkspaceStatus.STOPPED; - } - return state.status; - } - } - - private MessageConsumer getEnvironmentLogger(String workspaceId) throws ServerException { - WebsocketMessageConsumer envMessageConsumer = - new WebsocketMessageConsumer<>(format(ENVIRONMENT_OUTPUT_CHANNEL_TEMPLATE, workspaceId)); - return new AbstractMessageConsumer() { - @Override - public void consume(MachineLogMessage message) throws IOException { - envMessageConsumer.consume(message); - } - }; + public Set getRuntimesIds() { + return new HashSet<>(states.keySet()); } /** - * Removes all workspaces from the in-memory storage, while + * Removes all states from the in-memory storage, while * {@link CheEnvironmentEngine} is responsible for environment destroying. */ @PreDestroy @@ -579,13 +587,13 @@ void cleanup() { sharedPool.terminateAndWait(); List idsToStop; - try (@SuppressWarnings("unused") CloseableLock l = locks.acquireWriteAllLock()) { - idsToStop = workspaces.entrySet() - .stream() - .filter(e -> e.getValue().status != STOPPING) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - workspaces.clear(); + try (@SuppressWarnings("unused") Unlocker u = locks.writeAllLock()) { + idsToStop = states.entrySet() + .stream() + .filter(e -> e.getValue().status != STOPPING) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + states.clear(); } // nothing to stop @@ -593,7 +601,7 @@ void cleanup() { return; } - LOG.info("Shutdown running workspaces, workspaces to shutdown '{}'", idsToStop.size()); + LOG.info("Shutdown running states, states to shutdown '{}'", idsToStop.size()); ExecutorService executor = Executors.newFixedThreadPool(2 * Runtime.getRuntime().availableProcessors(), new ThreadFactoryBuilder().setNameFormat("StopEnvironmentsPool-%d") @@ -603,6 +611,8 @@ void cleanup() { executor.execute(() -> { try { envEngine.stop(id); + } catch (EnvironmentNotRunningException ignored) { + // could be stopped during workspace pool shutdown } catch (Exception x) { LOG.error(x.getMessage(), x); } @@ -623,21 +633,26 @@ void cleanup() { } } - private void ensurePreDestroyIsNotExecuted() throws ServerException { + private void ensurePreDestroyIsNotExecuted() { if (isPreDestroyInvoked) { - throw new ServerException("Could not perform operation because application server is stopping"); + throw new IllegalStateException("Could not perform operation because application server is stopping"); } } - private WorkspaceState getRunningState(String workspaceId) throws NotFoundException, ConflictException { - final WorkspaceState state = workspaces.get(workspaceId); + private RuntimeState getExistingState(String workspaceId) throws NotFoundException { + RuntimeState state = states.get(workspaceId); if (state == null) { throw new NotFoundException("Workspace with id '" + workspaceId + "' is not running"); } - if (state.getStatus() != RUNNING) { + return state; + } + + private RuntimeState getRunningState(String workspaceId) throws NotFoundException, ConflictException { + RuntimeState state = getExistingState(workspaceId); + if (state.status != RUNNING) { throw new ConflictException(format("Workspace with id '%s' is not 'RUNNING', it's status is '%s'", workspaceId, - state.getStatus())); + state.status)); } return state; } @@ -645,11 +660,12 @@ private WorkspaceState getRunningState(String workspaceId) throws NotFoundExcept protected void launchAgents(Instance instance, List agents) throws ServerException { try { for (AgentKey agentKey : agentSorter.sort(agents)) { - LOG.info("Launching '{}' agent at workspace {}", agentKey.getId(), instance.getWorkspaceId()); - - Agent agent = agentRegistry.getAgent(agentKey); - AgentLauncher launcher = launcherFactory.find(agentKey.getId(), instance.getConfig().getType()); - launcher.launch(instance, agent); + if (!Thread.currentThread().isInterrupted()) { + LOG.info("Launching '{}' agent at workspace {}", agentKey.getId(), instance.getWorkspaceId()); + Agent agent = agentRegistry.getAgent(agentKey); + AgentLauncher launcher = launcherFactory.find(agentKey.getId(), instance.getConfig().getType()); + launcher.launch(instance, agent); + } } } catch (AgentException e) { throw new MachineException(e.getMessage(), e); @@ -657,95 +673,125 @@ protected void launchAgents(Instance instance, List agents) throws Serve } /** - * Initializes workspace in {@link WorkspaceStatus#STARTING} status, - * saves the state or throws an appropriate exception if the workspace is already initialized. + * Starts the environment publishing all the necessary events. + * Respects task interruption & stops the workspace if starting task is cancelled. */ - private void initState(String workspaceId, String workspaceName, String envName) throws ConflictException, ServerException { - try (CloseableLock ignored = locks.acquireWriteLock(workspaceId)) { + private void startEnvironmentAndPublishEvents(EnvironmentImpl environment, + String workspaceId, + String envName, + boolean recover) throws ServerException, + EnvironmentException, + ConflictException { + try { + envEngine.start(workspaceId, + envName, + environment, + recover, + new WebsocketMessageConsumer<>(format(ENVIRONMENT_OUTPUT_CHANNEL_TEMPLATE, workspaceId)), + new MachineAgentsLauncher(environment.getMachines())); + } catch (EnvironmentStartInterruptedException x) { + // environment start was interrupted, it's either shutdown or direct stop + // in the case of shutdown make sure the status is correct, + // otherwise workspace is already stopping + compareAndSetStatus(workspaceId, WorkspaceStatus.STARTING, WorkspaceStatus.STOPPING); + removeStateAndPublishStopEvents(workspaceId); + throw x; + } catch (EnvironmentException | ServerException | ConflictException x) { + // environment can't be started for some reason, STARTING -> STOPPED + removeState(workspaceId); + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withEventType(EventType.ERROR) + .withPrevStatus(WorkspaceStatus.STARTING) + .withStatus(WorkspaceStatus.STOPPED) + .withError("Start of environment '" + envName + "' failed. Error: " + x.getMessage())); + throw x; + } + + // disallow direct start cancellation, STARTING -> RUNNING + WorkspaceStatus prevStatus; + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { ensurePreDestroyIsNotExecuted(); - final WorkspaceState state = workspaces.get(workspaceId); - if (state != null) { - throw new ConflictException(format("Could not start workspace '%s' because its status is '%s'", - workspaceName, - state.status)); + RuntimeState state = states.get(workspaceId); + prevStatus = state.status; + if (state.status == WorkspaceStatus.STARTING) { + state.status = WorkspaceStatus.RUNNING; + state.startTask = null; + state.startFuture = null; + } + } + + // either current thread is interrupted right after status update, + // or stop is called directly, anyway stop the environment + if (Thread.interrupted() || prevStatus != WorkspaceStatus.STARTING) { + try { + stopEnvironmentAndPublishEvents(workspaceId, WorkspaceStatus.STARTING); + } catch (Exception x) { + LOG.error("Couldn't stop the environment '{}' of the workspace '{}'. Error: {}", + envName, + workspaceId, + x.getMessage()); } - workspaces.put(workspaceId, new WorkspaceState(WorkspaceStatus.STARTING, envName)); + throw new EnvironmentStartInterruptedException(workspaceId, envName); } + + // normally started, notify clients + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withStatus(WorkspaceStatus.RUNNING) + .withEventType(EventType.RUNNING) + .withPrevStatus(WorkspaceStatus.STARTING)); } - /** Starts the machine instances. */ - private void doStart(EnvironmentImpl environment, - String workspaceId, - String envName, - boolean recover) throws ServerException { + /** STOPPING -> remove runtime -> STOPPED. */ + private void removeStateAndPublishStopEvents(String workspaceId) { eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) .withWorkspaceId(workspaceId) - .withStatus(WorkspaceStatus.STARTING) - .withEventType(EventType.STARTING) - .withPrevStatus(WorkspaceStatus.STOPPED)); + .withPrevStatus(STARTING) + .withStatus(WorkspaceStatus.STOPPING) + .withEventType(EventType.STOPPING)); + removeState(workspaceId); + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withPrevStatus(WorkspaceStatus.STOPPING) + .withEventType(EventType.STOPPED) + .withStatus(WorkspaceStatus.STOPPED)); + } + /** + * Stops the workspace publishing all the necessary events. + */ + private void stopEnvironmentAndPublishEvents(String workspaceId, + WorkspaceStatus prevStatus) throws ServerException, + EnvironmentException { + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withPrevStatus(prevStatus) + .withStatus(WorkspaceStatus.STOPPING) + .withEventType(EventType.STOPPING)); + removeState(workspaceId); try { - List machines = envEngine.start(workspaceId, - envName, - environment, - recover, - getEnvironmentLogger(workspaceId)); - launchAgents(environment, machines); - - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireWriteLock(workspaceId)) { - ensurePreDestroyIsNotExecuted(); - WorkspaceState workspaceState = workspaces.get(workspaceId); - workspaceState.status = WorkspaceStatus.RUNNING; - } - - eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspaceId) - .withStatus(WorkspaceStatus.RUNNING) - .withEventType(EventType.RUNNING) - .withPrevStatus(WorkspaceStatus.STARTING)); - } catch (ApiException | EnvironmentException | RuntimeException e) { - try { - envEngine.stop(workspaceId); - } catch (EnvironmentNotRunningException ignore) { - } catch (Exception ex) { - LOG.error(ex.getLocalizedMessage(), ex); - } - String environmentStartError = "Start of environment " + envName + - " failed. Error: " + e.getLocalizedMessage(); - try (@SuppressWarnings("unused") CloseableLock lock = locks.acquireWriteLock(workspaceId)) { - workspaces.remove(workspaceId); - } + envEngine.stop(workspaceId); + } catch (Exception x) { eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) .withWorkspaceId(workspaceId) + .withPrevStatus(WorkspaceStatus.STOPPING) .withEventType(EventType.ERROR) - .withPrevStatus(WorkspaceStatus.STARTING) - .withError(environmentStartError)); - - throw new ServerException(environmentStartError, e); - } - } - - private void launchAgents(EnvironmentImpl environment, List machines) throws ServerException { - for (Instance instance : machines) { - Map envMachines = environment.getMachines(); - if (envMachines != null) { - ExtendedMachine extendedMachine = envMachines.get(instance.getConfig().getName()); - if (extendedMachine != null) { - List agents = extendedMachine.getAgents(); - launchAgents(instance, agents); - } + .withError(x.getMessage()) + .withStatus(WorkspaceStatus.STOPPED)); + try { + throw x; + } catch (ServerException rethrow) { + throw rethrow; + } catch (Exception wrap) { + throw new ServerException(wrap.getMessage(), wrap); } } - } - - private static EnvironmentImpl copyEnv(Workspace workspace, String envName) { - final Environment environment = workspace.getConfig().getEnvironments().get(envName); - if (environment == null) { - throw new IllegalArgumentException(format("Workspace '%s' doesn't contain environment '%s'", - workspace.getId(), - envName)); - } - return new EnvironmentImpl(environment); + eventsService.publish(DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withPrevStatus(WorkspaceStatus.STOPPING) + .withEventType(EventType.STOPPED) + .withStatus(WorkspaceStatus.STOPPED)); } /** @@ -754,9 +800,10 @@ private static EnvironmentImpl copyEnv(Workspace workspace, String envName) { * Returns true if the status of workspace was updated with {@code to} value. */ private boolean compareAndSetStatus(String id, WorkspaceStatus from, WorkspaceStatus to) { - try (@SuppressWarnings("unused") CloseableLock l = locks.acquireWriteLock(id)) { - WorkspaceState state = workspaces.get(id); - if (state != null && state.getStatus() == from) { + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(id)) { + ensurePreDestroyIsNotExecuted(); + RuntimeState state = states.get(id); + if (state != null && state.status == from) { state.status = to; return true; } @@ -764,7 +811,15 @@ private boolean compareAndSetStatus(String id, WorkspaceStatus from, WorkspaceSt return false; } - /** Creates a snapshot and changes status SNAPSHOTTING -> RUNNING . */ + /** Removes state from in-memory storage in write lock. */ + private void removeState(String workspaceId) { + try (@SuppressWarnings("unused") Unlocker u = locks.writeLock(workspaceId)) { + ensurePreDestroyIsNotExecuted(); + states.remove(workspaceId); + } + } + + /** Creates a snapshot and changes status SNAPSHOTTING -> RUNNING. */ private void snapshotAndUpdateStatus(String workspaceId) throws NotFoundException, ConflictException, ServerException { @@ -774,7 +829,7 @@ private void snapshotAndUpdateStatus(String workspaceId) throws NotFoundExceptio .withEventType(EventType.SNAPSHOT_CREATING) .withPrevStatus(WorkspaceStatus.RUNNING)); - WorkspaceRuntimeImpl runtime = get(workspaceId).getRuntime(); + WorkspaceRuntimeImpl runtime = getRuntime(workspaceId); List machines = runtime.getMachines(); machines.sort(comparing(m -> !m.getConfig().isDev(), Boolean::compare)); @@ -831,97 +886,155 @@ private void snapshotAndUpdateStatus(String workspaceId) throws NotFoundExceptio .withPrevStatus(WorkspaceStatus.SNAPSHOTTING)); } - public static class WorkspaceState { - private WorkspaceStatus status; - private String activeEnv; - - public WorkspaceState(WorkspaceStatus status, String activeEnv) { - this.status = status; - this.activeEnv = activeEnv; + /** Holds runtime information while workspace is running. */ + @VisibleForTesting + static class RuntimeState { + + WorkspaceStatus status; + String envName; + StartTask startTask; + Future startFuture; + + RuntimeState(RuntimeState state) { + this.status = state.status; + this.envName = state.envName; + this.startFuture = state.startFuture; + this.startTask = state.startTask; } - public String getActiveEnv() { - return activeEnv; + RuntimeState(WorkspaceStatus status, + String envName, + StartTask startTask, + Future startFuture) { + this.status = status; + this.envName = envName; + this.startTask = startTask; + this.startFuture = startFuture; } + } - public WorkspaceStatus getStatus() { - return status; + @VisibleForTesting + class StartTask implements Callable { + + final String workspaceId; + final String envName; + final EnvironmentImpl environment; + final boolean recover; + final CompletableFuture cmpFuture; + final AtomicBoolean used; + final CountDownLatch allowStartLatch; + final CountDownLatch completionLatch; + + volatile Exception exception; + + StartTask(String workspaceId, + String envName, + EnvironmentImpl environment, + boolean recover, + CompletableFuture cmpFuture) { + this.workspaceId = workspaceId; + this.envName = envName; + this.environment = environment; + this.recover = recover; + this.cmpFuture = cmpFuture; + this.used = new AtomicBoolean(false); + this.completionLatch = new CountDownLatch(1); + this.allowStartLatch = new CountDownLatch(1); } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof WorkspaceState)) return false; - WorkspaceState that = (WorkspaceState)o; - return status == that.status && - Objects.equals(activeEnv, that.activeEnv); + public WorkspaceRuntimeImpl call() throws Exception { + if (!markAsUsed()) { + throw new CancellationException(format("Start of the workspace '%s' was cancelled", workspaceId)); + } + allowStartLatch.await(); + try { + startEnvironmentAndPublishEvents(environment, workspaceId, envName, recover); + WorkspaceRuntimeImpl runtime = getRuntime(workspaceId); + cmpFuture.complete(runtime); + return runtime; + } catch (IllegalStateException illegalStateEx) { + if (isPreDestroyInvoked) { + exception = new EnvironmentStartInterruptedException(workspaceId, envName); + } else { + exception = new ServerException(illegalStateEx.getMessage(), illegalStateEx); + } + cmpFuture.completeExceptionally(exception); + throw exception; + } catch (Exception occurred) { + cmpFuture.completeExceptionally(exception = occurred); + throw occurred; + } finally { + completionLatch.countDown(); + } } - @Override - public int hashCode() { - return Objects.hash(status, activeEnv); - } - } - - /** - * Wrapper for the {@link WorkspaceRuntime} instance. - * Knows the state of the started workspace runtime, - * helps to postpone {@code WorkspaceRuntime} instance creation to - * the time when all the machines from the workspace are created. - */ - public static class RuntimeDescriptor { - - private WorkspaceRuntimeImpl runtime; - private WorkspaceStatus status; - - public RuntimeDescriptor(WorkspaceStatus workspaceStatus, - WorkspaceRuntimeImpl runtime) { - this.status = workspaceStatus; - this.runtime = runtime; + /** + * Awaits this task to complete, rethrows exceptions occurred during the invocation. + */ + void await() throws InterruptedException, + ServerException, + ConflictException, + EnvironmentException { + completionLatch.await(); + if (exception != null) { + try { + throw exception; + } catch (ServerException | EnvironmentException | ConflictException rethrow) { + throw rethrow; + } catch (Exception x) { + throw new ServerException(x.getMessage(), x); + } + } } - /** Returns the instance of {@code WorkspaceRuntime} described by this descriptor. */ - public WorkspaceRuntimeImpl getRuntime() { - return runtime; + /** + * Completes corresponding completable future exceptionally + * with {@link EnvironmentStartInterruptedException}. + */ + void earlyComplete() { + exception = new EnvironmentStartInterruptedException(workspaceId, envName); + cmpFuture.completeExceptionally(exception); + completionLatch.countDown(); } - public void setRuntime(WorkspaceRuntimeImpl runtime) { - this.runtime = runtime; + /** + * Marks this task as used, returns true only if it was unused before. + */ + boolean markAsUsed() { + return used.compareAndSet(false, true); } /** - * Returns the status of the {@code WorkspaceRuntime} described by this descriptor. - * Never returns {@link WorkspaceStatus#STOPPED} status, you'll rather get {@link NotFoundException} - * from {@link #get(String)} method. + * Allows start of this task. + * The task caller will wait until this method is called. */ - public WorkspaceStatus getRuntimeStatus() { - return status; + void unlockStart() { + allowStartLatch.countDown(); } + } - private void setRuntimeStatus(WorkspaceStatus status) { - this.status = status; - } + private class MachineAgentsLauncher implements MachineStartedHandler { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof RuntimeDescriptor)) return false; - RuntimeDescriptor that = (RuntimeDescriptor)o; - return Objects.equals(runtime, that.runtime) && - status == that.status; + private final Map nameToMachine; + + private MachineAgentsLauncher(Map nameToMachine) { + this.nameToMachine = nameToMachine; } @Override - public int hashCode() { - return Objects.hash(runtime, status); + public void started(Instance machine) throws ServerException { + launchAgents(machine, nameToMachine.get(machine.getConfig().getName()).getAgents()); } + } - @Override - public String toString() { - return "RuntimeDescriptor{" + - "runtime=" + runtime + - ", status=" + status + - '}'; + private static EnvironmentImpl copyEnv(Workspace workspace, String envName) { + Environment environment = workspace.getConfig().getEnvironments().get(envName); + if (environment == null) { + throw new IllegalArgumentException(format("Workspace '%s' doesn't contain environment '%s'", + workspace.getId(), + envName)); } + return new EnvironmentImpl(environment); } } diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java index 0254978578d..fecee50a592 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java @@ -10,19 +10,25 @@ *******************************************************************************/ package org.eclipse.che.api.workspace.server; +import com.google.common.base.MoreObjects; +import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; +import org.eclipse.che.commons.annotation.Nullable; import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.PostConstruct; +import javax.inject.Named; import javax.inject.Singleton; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; /** @@ -35,12 +41,35 @@ public class WorkspaceSharedPool { private final ExecutorService executor; - public WorkspaceSharedPool() { - executor = Executors.newFixedThreadPool(2 * Runtime.getRuntime().availableProcessors(), - new ThreadFactoryBuilder().setNameFormat("WorkspaceSharedPool-%d") - .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) - .setDaemon(false) - .build()); + @Inject + public WorkspaceSharedPool(@Named("che.workspace.pool.type") String poolType, + @Named("che.workspace.pool.exact_size") @Nullable String exactSizeProp, + @Named("che.workspace.pool.cores_multiplier") @Nullable String coresMultiplierProp) { + ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("WorkspaceSharedPool-%d") + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setDaemon(false) + .build(); + switch (poolType.toLowerCase()) { + case "cached": + executor = Executors.newCachedThreadPool(factory); + break; + case "fixed": + Integer exactSize = exactSizeProp == null ? null : Ints.tryParse(exactSizeProp); + int size; + if (exactSize != null && exactSize > 0) { + size = exactSize; + } else { + size = Runtime.getRuntime().availableProcessors(); + Integer coresMultiplier = coresMultiplierProp == null ? null : Ints.tryParse(coresMultiplierProp); + if (coresMultiplier != null && coresMultiplier > 0) { + size *= coresMultiplier; + } + } + executor = Executors.newFixedThreadPool(size, factory); + break; + default: + throw new IllegalArgumentException("The type of the pool '" + poolType + "' is not supported"); + } } /** Returns an {@link ExecutorService} managed by this pool instance. */ @@ -66,7 +95,11 @@ public Future submit(Callable callable) { /** * Terminates this pool, may be called multiple times, - * waits until pool is terminated or timeout reached. + * waits until pool is terminated or timeout is reached. + * + *

Note that the method is not designed to be used from + * different threads, but the other components may use it in their + * post construct methods to ensure that all the tasks finished their execution. * * @return true if executor successfully terminated and false if not * terminated(either await termination timeout is reached or thread was interrupted) diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/WorkspaceRuntimeImpl.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/WorkspaceRuntimeImpl.java index 649fe2412f6..1f45f8d719c 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/WorkspaceRuntimeImpl.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/model/impl/WorkspaceRuntimeImpl.java @@ -34,8 +34,22 @@ public class WorkspaceRuntimeImpl implements WorkspaceRuntime { private MachineImpl devMachine; private List machines; - public WorkspaceRuntimeImpl(String activeEnv) { + public WorkspaceRuntimeImpl(String activeEnv, Collection machines) { this.activeEnv = activeEnv; + if (machines != null) { + this.machines = new ArrayList<>(machines.size()); + for (Machine machine : machines) { + if (machine.getConfig().isDev()) { + if (machine.getRuntime() != null) { + rootFolder = machine.getRuntime().projectsRoot(); + } + devMachine = new MachineImpl(machine); + this.machines.add(devMachine); + } else { + this.machines.add(new MachineImpl(machine)); + } + } + } } public WorkspaceRuntimeImpl(String activeEnv, @@ -47,9 +61,11 @@ public WorkspaceRuntimeImpl(String activeEnv, if (devMachine != null) { this.devMachine = new MachineImpl(devMachine); } - this.machines = machines.stream() - .map(MachineImpl::new) - .collect(toList()); + if (machines != null) { + this.machines = machines.stream() + .map(MachineImpl::new) + .collect(toList()); + } } public WorkspaceRuntimeImpl(WorkspaceRuntime runtime) { diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/environment/server/CheEnvironmentEngineTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/environment/server/CheEnvironmentEngineTest.java index bcfaf4317f4..9c1b4600922 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/environment/server/CheEnvironmentEngineTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/environment/server/CheEnvironmentEngineTest.java @@ -15,6 +15,7 @@ import org.eclipse.che.api.agent.shared.model.AgentKey; import org.eclipse.che.api.core.ConflictException; import org.eclipse.che.api.core.NotFoundException; +import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.core.model.machine.Machine; import org.eclipse.che.api.core.model.machine.MachineLogMessage; import org.eclipse.che.api.core.model.machine.MachineStatus; @@ -24,6 +25,7 @@ import org.eclipse.che.api.core.util.LineConsumer; import org.eclipse.che.api.core.util.MessageConsumer; import org.eclipse.che.api.environment.server.exception.EnvironmentNotRunningException; +import org.eclipse.che.api.environment.server.exception.EnvironmentStartInterruptedException; import org.eclipse.che.api.environment.server.model.CheServiceBuildContextImpl; import org.eclipse.che.api.environment.server.model.CheServiceImpl; import org.eclipse.che.api.environment.server.model.CheServicesEnvironmentImpl; @@ -65,6 +67,7 @@ import java.util.Optional; import java.util.UUID; +import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -88,6 +91,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; /** * @author Alexander Garagatyi @@ -105,23 +109,25 @@ public class CheEnvironmentEngineTest { @Mock private MachineInstanceProvider machineProvider; @Mock - private MachineInstanceProviders machineInstanceProviders; + private MachineInstanceProviders machineInstanceProviders; @Mock - private EventService eventService; + private EventService eventService; @Mock - private SnapshotDao snapshotDao; + private SnapshotDao snapshotDao; @Mock - private RecipeDownloader recipeDownloader; + private RecipeDownloader recipeDownloader; @Mock - InfrastructureProvisioner infrastructureProvisioner; + InfrastructureProvisioner infrastructureProvisioner; @Mock - private ContainerNameGenerator containerNameGenerator; + private ContainerNameGenerator containerNameGenerator; @Mock - private AgentRegistry agentRegistry; + private AgentRegistry agentRegistry; @Mock - private Agent agent; + private Agent agent; @Mock - private EnvironmentParser environmentParser; + private EnvironmentParser environmentParser; + @Mock + private MachineStartedHandler startedHandler; private CheEnvironmentEngine engine; @@ -241,10 +247,76 @@ public void shouldBeAbleToStartEnvironment() throws Exception { envName, env, false, - messageConsumer); + messageConsumer, + startedHandler); // then assertEquals(machines, expectedMachines); + for (Instance expectedMachine : expectedMachines) { + verify(startedHandler).started(expectedMachine); + } + } + + @Test + public void stopsTheEnvironmentWhileStartOfMachineIsInterrupted() throws Exception { + // given + EnvironmentImpl env = createEnv(); + String envName = "env-1"; + String workspaceId = "wsId"; + + int[] counter = new int[] {env.getMachines().size()}; + ArrayList created = new ArrayList<>(); + when(machineProvider.startService(anyString(), + eq(workspaceId), + eq(envName), + anyString(), + anyBoolean(), + anyString(), + any(CheServiceImpl.class), + any(LineConsumer.class))) + .thenAnswer(invocationOnMock -> { + // interrupt when the last machine from environment is started + if (--counter[0] == 0) { + Thread.currentThread().interrupt(); + throw new ServerException("interrupted!"); + } + Object[] arguments = invocationOnMock.getArguments(); + NoOpMachineInstance instance = spy(new NoOpMachineInstance(createMachine(workspaceId, + envName, + (CheServiceImpl)arguments[6], + (String)arguments[3], + (boolean)arguments[4]))); + created.add(instance); + return instance; + }); + when(environmentParser.parse(env)).thenReturn(createCheServicesEnv()); + + // when, then + try { + engine.start(workspaceId, + envName, + env, + false, + messageConsumer, + startedHandler); + fail("environment must not be running"); + } catch (EnvironmentStartInterruptedException x) { + assertEquals(x.getMessage(), format("Start of environment '%s' in workspace '%s' is interrupted", + envName, workspaceId)); + } + + // environment must not be running + try { + engine.getMachines(workspaceId); + fail("environment must not be running"); + } catch (EnvironmentNotRunningException x) { + assertEquals(x.getMessage(), format("Environment with ID '%s' is not found", workspaceId)); + } + + // all the machines expect of the last one must be destroyed + for (Instance instance : created) { + verify(instance).destroy(); + } } @Test @@ -471,10 +543,10 @@ public void shouldBeAbleToStartEnvironmentWithRecover() throws Exception { // when List machines = engine.start(workspaceId, - envName, - env, - true, - messageConsumer); + envName, + env, + true, + messageConsumer); // then assertEquals(machines, expectedMachines); diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java index a68d6e3d4a2..129cd4d4f28 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceManagerTest.java @@ -16,6 +16,7 @@ import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.core.model.machine.MachineStatus; +import org.eclipse.che.api.core.model.workspace.Workspace; import org.eclipse.che.api.core.model.workspace.WorkspaceConfig; import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; import org.eclipse.che.api.core.notification.EventService; @@ -27,7 +28,6 @@ import org.eclipse.che.api.machine.server.model.impl.MachineSourceImpl; import org.eclipse.che.api.machine.server.model.impl.SnapshotImpl; import org.eclipse.che.api.machine.server.spi.SnapshotDao; -import org.eclipse.che.api.workspace.server.WorkspaceRuntimes.RuntimeDescriptor; import org.eclipse.che.api.workspace.server.model.impl.EnvironmentImpl; import org.eclipse.che.api.workspace.server.model.impl.EnvironmentRecipeImpl; import org.eclipse.che.api.workspace.server.model.impl.ExtendedMachineImpl; @@ -52,23 +52,26 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import static com.google.common.base.Strings.isNullOrEmpty; -import static com.google.common.util.concurrent.Futures.immediateFuture; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.RUNNING; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STARTING; import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STOPPED; +import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STOPPING; import static org.eclipse.che.api.workspace.server.WorkspaceManager.CREATED_ATTRIBUTE_NAME; import static org.eclipse.che.api.workspace.server.WorkspaceManager.UPDATED_ATTRIBUTE_NAME; import static org.eclipse.che.api.workspace.shared.Constants.AUTO_CREATE_SNAPSHOT; import static org.eclipse.che.api.workspace.shared.Constants.AUTO_RESTORE_FROM_SNAPSHOT; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; @@ -79,7 +82,6 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; /** @@ -164,7 +166,7 @@ public void getsWorkspaceByIdWithoutRuntime() throws Exception { @Test public void getsWorkspaceByIdWithRuntime() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, STARTING); + mockRuntime(workspace, STARTING); WorkspaceImpl result = workspaceManager.getWorkspace(workspace.getId()); @@ -206,7 +208,7 @@ public void shouldBeAbleToGetWorkspaceByKeyWithoutOwner() throws Exception { } @Test - public void shouldBeAbleToGetWorkspacesAvailableForUserWithRuntimes() throws Exception { + public void shouldBeAbleToGetWorkspacesAvailableForUser() throws Exception { // given final WorkspaceConfig config = createConfig(); @@ -214,9 +216,8 @@ public void shouldBeAbleToGetWorkspacesAvailableForUserWithRuntimes() throws Exc final WorkspaceImpl workspace2 = createAndMockWorkspace(config, NAMESPACE_2); when(workspaceDao.getWorkspaces(NAMESPACE)).thenReturn(asList(workspace1, workspace2)); - final RuntimeDescriptor descriptor = createDescriptor(workspace2, RUNNING); - when(runtimes.get(workspace2.getId())).thenReturn(descriptor); - when(runtimes.get(workspace1.getId())).thenThrow(new NotFoundException("no runtime")); + mockRuntime(workspace1, STOPPED); + mockRuntime(workspace2, RUNNING); // when final List result = workspaceManager.getWorkspaces(NAMESPACE, true); @@ -225,68 +226,19 @@ public void shouldBeAbleToGetWorkspacesAvailableForUserWithRuntimes() throws Exc assertEquals(result.size(), 2); final WorkspaceImpl res1 = result.get(0); - assertEquals(res1.getStatus(), STOPPED, "Workspace status wasn't changed from STARTING to STOPPED"); - assertNull(res1.getRuntime(), "Workspace has unexpected runtime"); + assertEquals(res1.getStatus(), STOPPED); assertFalse(res1.isTemporary(), "Workspace must be permanent"); final WorkspaceImpl res2 = result.get(1); assertEquals(res2.getStatus(), RUNNING, "Workspace status wasn't changed to the runtime instance status"); - assertEquals(res2.getRuntime(), descriptor.getRuntime(), "Workspace doesn't have expected runtime"); assertFalse(res2.isTemporary(), "Workspace must be permanent"); } @Test - public void shouldBeAbleToGetWorkspacesAvailableForUserWithoutRuntimes() throws Exception { - // given - final WorkspaceConfig config = createConfig(); - - final WorkspaceImpl workspace1 = createAndMockWorkspace(config, NAMESPACE); - final WorkspaceImpl workspace2 = createAndMockWorkspace(config, NAMESPACE_2); - - when(workspaceDao.getWorkspaces(NAMESPACE)).thenReturn(asList(workspace1, workspace2)); - when(runtimes.getStatus(workspace2.getId())).thenReturn(RUNNING); - when(runtimes.getStatus(workspace1.getId())).thenReturn(STOPPED); - - // when - final List result = workspaceManager.getWorkspaces(NAMESPACE, false); - - // then - assertEquals(result.size(), 2); - - final WorkspaceImpl res1 = result.get(0); - assertEquals(res1.getStatus(), STOPPED, "Workspace status wasn't changed from STARTING to STOPPED"); - assertNull(res1.getRuntime(), "Workspace has unexpected runtime"); - assertFalse(res1.isTemporary(), "Workspace must be permanent"); - - final WorkspaceImpl res2 = result.get(1); - assertEquals(res2.getStatus(), RUNNING, "Workspace status wasn't changed to the runtime instance status"); - assertNull(res1.getRuntime(), "Workspace has unexpected runtime"); - assertFalse(res2.isTemporary(), "Workspace must be permanent"); - } - - @Test - public void shouldBeAbleToGetWorkspacesByNamespaceWithoutRuntimes() throws Exception { + public void shouldBeAbleToGetWorkspacesByNamespace() throws Exception { // given final WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, RUNNING); - - // when - final List result = workspaceManager.getByNamespace(workspace.getNamespace(), false); - - // then - assertEquals(result.size(), 1); - - final WorkspaceImpl res1 = result.get(0); - assertEquals(res1.getStatus(), RUNNING, "Workspace status wasn't changed to the runtime instance status"); - assertNull(res1.getRuntime(), "workspace has unexpected runtime"); - assertFalse(res1.isTemporary(), "Workspace must be permanent"); - } - - @Test - public void shouldBeAbleToGetWorkspacesByNamespaceWithRuntimes() throws Exception { - // given - final WorkspaceImpl workspace = createAndMockWorkspace(); - final RuntimeDescriptor descriptor = createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); // when final List result = workspaceManager.getByNamespace(workspace.getNamespace(), true); @@ -296,14 +248,13 @@ public void shouldBeAbleToGetWorkspacesByNamespaceWithRuntimes() throws Exceptio final WorkspaceImpl res1 = result.get(0); assertEquals(res1.getStatus(), RUNNING, "Workspace status wasn't changed to the runtime instance status"); - assertEquals(res1.getRuntime(), descriptor.getRuntime(), "Workspace doesn't have expected runtime"); assertFalse(res1.isTemporary(), "Workspace must be permanent"); } @Test public void getWorkspaceByNameShouldReturnWorkspaceWithStatusEqualToItsRuntimeStatus() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, STARTING); + mockRuntime(workspace, STARTING); final WorkspaceImpl result = workspaceManager.getWorkspace(workspace.getConfig().getName(), workspace.getNamespace()); @@ -325,7 +276,7 @@ public void shouldBeAbleToUpdateWorkspace() throws Exception { @Test public void workspaceUpdateShouldReturnWorkspaceWithStatusEqualToItsRuntimeStatus() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, STARTING); + mockRuntime(workspace, STARTING); final WorkspaceImpl updated = workspaceManager.updateWorkspace(workspace.getId(), workspace); @@ -352,6 +303,7 @@ public void shouldNotRemoveWorkspaceIfItIsNotStopped() throws Exception { @Test public void shouldBeAbleToStartWorkspaceById() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), workspace.getConfig().getDefaultEnv(), @@ -365,6 +317,7 @@ public void shouldBeAbleToStartWorkspaceById() throws Exception { public void shouldRecoverWorkspaceWhenRecoverParameterIsNullAndAutoRestoreAttributeIsSetAndSnapshotExists() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_RESTORE_FROM_SNAPSHOT, "true"); + mockStart(workspace); SnapshotImpl.SnapshotBuilder snapshotBuilder = SnapshotImpl.builder() .generateId() .setEnvName("env") @@ -392,6 +345,7 @@ public void shouldRecoverWorkspaceWhenRecoverParameterIsNullAndAutoRestoreAttrib @Test public void shouldRecoverWorkspaceWhenRecoverParameterIsTrueAndSnapshotExists() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); + mockStart(workspace); SnapshotImpl.SnapshotBuilder snapshotBuilder = SnapshotImpl.builder() .generateId() .setEnvName("env") @@ -421,6 +375,7 @@ public void shouldNotRecoverWorkspaceWhenRecoverParameterIsNullAndAutoRestoreAtt final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_RESTORE_FROM_SNAPSHOT, "true"); when(workspaceDao.get(workspace.getId())).thenReturn(workspace); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), workspace.getConfig().getDefaultEnv(), @@ -433,6 +388,7 @@ public void shouldNotRecoverWorkspaceWhenRecoverParameterIsNullAndAutoRestoreAtt @Test public void shouldNotRecoverWorkspaceWhenRecoverParameterIsTrueButSnapshotDoesNotExist() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), workspace.getConfig().getDefaultEnv(), @@ -446,6 +402,7 @@ public void shouldNotRecoverWorkspaceWhenRecoverParameterIsTrueButSnapshotDoesNo public void shouldNotRecoverWorkspaceWhenRecoverParameterIsFalseAndAutoRestoreAttributeIsSetAndSnapshotExists() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_RESTORE_FROM_SNAPSHOT, "true"); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), workspace.getConfig().getDefaultEnv(), @@ -458,8 +415,7 @@ public void shouldNotRecoverWorkspaceWhenRecoverParameterIsFalseAndAutoRestoreAt @Test public void workspaceStartShouldUseDefaultEnvIfNullEnvNameProvided() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); - final RuntimeDescriptor descriptor = createDescriptor(workspace, STARTING); - when(runtimes.startAsync(any(), anyString(), anyBoolean())).thenReturn(immediateFuture(descriptor)); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), null, null); @@ -471,9 +427,7 @@ public void usesProvidedEnvironmentInsteadOfDefault() throws Exception { WorkspaceConfigImpl config = createConfig(); config.getEnvironments().put("non-default-env", new EnvironmentImpl(null, null)); WorkspaceImpl workspace = createAndMockWorkspace(config, NAMESPACE); - - RuntimeDescriptor descriptor = createDescriptor(workspace, STARTING); - when(runtimes.startAsync(any(), anyString(), anyBoolean())).thenReturn(immediateFuture(descriptor)); + mockStart(workspace); workspaceManager.startWorkspace(workspace.getId(), "non-default-env", false); @@ -491,8 +445,7 @@ public void startShouldThrowNotFoundExceptionWhenProvidedEnvDoesNotExist() throw @Test public void shouldBeAbleToStartTemporaryWorkspace() throws Exception { - when(runtimes.start(any(), anyString(), anyBoolean())).thenReturn(mock(RuntimeDescriptor.class)); - when(runtimes.get(any())).thenThrow(new NotFoundException("")); + mockAnyWorkspaceStart(); workspaceManager.startWorkspace(createConfig(), NAMESPACE, true); @@ -504,7 +457,7 @@ public void shouldBeAbleToStartTemporaryWorkspace() throws Exception { @Test public void shouldBeAbleToStopWorkspace() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(createConfig(), NAMESPACE); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); // when workspaceManager.stopWorkspace(workspace.getId()); @@ -522,7 +475,7 @@ public void shouldBeAbleToStopWorkspace() throws Exception { @Test public void createsSnapshotBeforeStoppingWorkspace() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); workspaceManager.stopWorkspace(workspace.getId(), true); @@ -531,11 +484,11 @@ public void createsSnapshotBeforeStoppingWorkspace() throws Exception { } @Test(expectedExceptions = ConflictException.class, - expectedExceptionsMessageRegExp = "Could not stop the workspace " + - "'.*' because its status is 'STARTING'.") + expectedExceptionsMessageRegExp = "Could not stop the workspace 'test-namespace:dev-workspace' because its " + + "status is 'STOPPING'. Workspace must be either 'STARTING' or 'RUNNING'") public void failsToStopNotRunningWorkspace() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, STARTING); + mockRuntime(workspace, STOPPING); workspaceManager.stopWorkspace(workspace.getId()); } @@ -543,7 +496,7 @@ public void failsToStopNotRunningWorkspace() throws Exception { @Test public void shouldStopWorkspaceEventIfSnapshotCreationFailed() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); doThrow(new ServerException("Test")).when(runtimes).snapshot(workspace.getId()); workspaceManager.stopWorkspace(workspace.getId(), true); @@ -556,7 +509,7 @@ public void shouldStopWorkspaceEventIfSnapshotCreationFailed() throws Exception public void shouldRemoveTemporaryWorkspaceAfterStop() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); workspace.setTemporary(true); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); workspaceManager.stopWorkspace(workspace.getId()); @@ -568,7 +521,7 @@ public void shouldRemoveTemporaryWorkspaceAfterStop() throws Exception { public void shouldRemoveTemporaryWorkspaceAfterStartFailed() throws Exception { WorkspaceImpl workspace = createAndMockWorkspace(); workspace.setTemporary(true); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); doThrow(new ServerException("")).when(runtimes).stop(workspace.getId()); workspaceManager.stopWorkspace(workspace.getId()); @@ -598,7 +551,7 @@ public void shouldBeAbleToGetSnapshots() throws Exception { public void shouldNotCreateSnapshotIfWorkspaceIsTemporaryAndAutoCreateSnapshotActivated() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(Constants.AUTO_CREATE_SNAPSHOT, "true"); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); SnapshotImpl oldSnapshot = mock(SnapshotImpl.class); when(snapshotDao.getSnapshot(eq(workspace.getId()), @@ -618,7 +571,7 @@ public void shouldNotCreateSnapshotIfWorkspaceIsTemporaryAndAutoCreateSnapshotAc public void shouldNotCreateSnapshotIfWorkspaceIsTemporaryAndAutoCreateSnapshotDisactivated() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(Constants.AUTO_CREATE_SNAPSHOT, "false"); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); SnapshotImpl oldSnapshot = mock(SnapshotImpl.class); when(snapshotDao.getSnapshot(eq(workspace.getId()), @@ -647,7 +600,7 @@ public void shouldCreateWorkspaceSnapshotUsingDefaultValueForAutoRestore() throw sharedPool); final WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); SnapshotImpl oldSnapshot = mock(SnapshotImpl.class); when(snapshotDao.getSnapshot(eq(workspace.getId()), @@ -675,6 +628,7 @@ public void shouldStartWorkspaceFromSnapshotUsingDefaultValueForAutoRestore() th snapshotDao, sharedPool); WorkspaceImpl workspace = createAndMockWorkspace(); + mockStart(workspace); SnapshotImpl.SnapshotBuilder snapshotBuilder = SnapshotImpl.builder() .generateId() @@ -769,9 +723,9 @@ public void shouldRemoveMachinesSnapshotsEvenSomeRemovalFails() throws Exception public void shouldBeAbleToStartMachineInRunningWs() throws Exception { // given WorkspaceImpl workspace = createAndMockWorkspace(); - RuntimeDescriptor descriptor = createAndMockDescriptor(workspace, RUNNING); + WorkspaceRuntimeImpl runtime = mockRuntime(workspace, RUNNING); MachineConfigImpl machineConfig = createMachine(workspace.getId(), - descriptor.getRuntime().getActiveEnv(), + runtime.getActiveEnv(), false).getConfig(); // when @@ -796,7 +750,7 @@ public void shouldThrowExceptionOnStartMachineInNonRunningWs() throws Exception public void shouldBeAbleToCreateSnapshot() throws Exception { // then WorkspaceImpl workspace = createAndMockWorkspace(); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); SnapshotImpl oldSnapshot = mock(SnapshotImpl.class); when(snapshotDao.getSnapshot(eq(workspace.getId()), eq(workspace.getConfig().getDefaultEnv()), @@ -814,8 +768,8 @@ public void shouldBeAbleToCreateSnapshot() throws Exception { public void shouldBeAbleToStopMachine() throws Exception { // given final WorkspaceImpl workspace = createAndMockWorkspace(); - RuntimeDescriptor descriptor = createAndMockDescriptor(workspace, RUNNING); - MachineImpl machine = descriptor.getRuntime().getMachines().get(0); + WorkspaceRuntimeImpl runtime = mockRuntime(workspace, RUNNING); + MachineImpl machine = runtime.getMachines().get(0); // when workspaceManager.stopMachine(workspace.getId(), machine.getId()); @@ -838,8 +792,8 @@ public void shouldNotStopMachineIfWorkspaceIsNotRunning() throws Exception { public void shouldBeAbleToGetMachineInstanceIfWorkspaceIsRunning() throws Exception { // given final WorkspaceImpl workspace = createAndMockWorkspace(); - RuntimeDescriptor descriptor = createAndMockDescriptor(workspace, RUNNING); - MachineImpl machine = descriptor.getRuntime().getMachines().get(0); + WorkspaceRuntimeImpl runtime = mockRuntime(workspace, RUNNING); + MachineImpl machine = runtime.getMachines().get(0); // when workspaceManager.getMachineInstance(workspace.getId(), machine.getId()); @@ -852,8 +806,8 @@ public void shouldBeAbleToGetMachineInstanceIfWorkspaceIsRunning() throws Except public void shouldBeAbleToGetMachineInstanceIfWorkspaceIsStarting() throws Exception { // given final WorkspaceImpl workspace = createAndMockWorkspace(); - RuntimeDescriptor descriptor = createAndMockDescriptor(workspace, STARTING); - MachineImpl machine = descriptor.getRuntime().getMachines().get(0); + WorkspaceRuntimeImpl runtime = mockRuntime(workspace, STARTING); + MachineImpl machine = runtime.getMachines().get(0); // when workspaceManager.getMachineInstance(workspace.getId(), machine.getId()); @@ -866,7 +820,7 @@ public void shouldBeAbleToGetMachineInstanceIfWorkspaceIsStarting() throws Excep public void passedCreateSnapshotParameterIsUsedInPreferenceToAttribute() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_CREATE_SNAPSHOT, "true"); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); workspaceManager.stopWorkspace(workspace.getId(), false); @@ -878,7 +832,7 @@ public void passedCreateSnapshotParameterIsUsedInPreferenceToAttribute() throws public void passedNullCreateSnapshotParameterIsIgnored() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_CREATE_SNAPSHOT, "true"); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); workspaceManager.stopWorkspace(workspace.getId(), null); @@ -890,7 +844,7 @@ public void passedNullCreateSnapshotParameterIsIgnored() throws Exception { public void passedFalseCreateSnapshotParameterIsUsedInPreferenceToAttribute() throws Exception { final WorkspaceImpl workspace = createAndMockWorkspace(); workspace.getAttributes().put(AUTO_CREATE_SNAPSHOT, "true"); - createAndMockDescriptor(workspace, RUNNING); + mockRuntime(workspace, RUNNING); workspaceManager.stopWorkspace(workspace.getId(), false); @@ -903,32 +857,20 @@ private void captureAsyncTaskAndExecuteSynchronously() { taskCaptor.getValue().run(); } - private RuntimeDescriptor createAndMockDescriptor(WorkspaceImpl workspace, WorkspaceStatus status) - throws ServerException, NotFoundException, ConflictException { - RuntimeDescriptor descriptor = createDescriptor(workspace, status); - when(runtimes.get(workspace.getId())).thenReturn(descriptor); + private WorkspaceRuntimeImpl mockRuntime(WorkspaceImpl workspace, WorkspaceStatus status) { when(runtimes.getStatus(workspace.getId())).thenReturn(status); - return descriptor; - } - - private RuntimeDescriptor createDescriptor(WorkspaceImpl workspace, WorkspaceStatus status) - throws ServerException, NotFoundException, ConflictException { - EnvironmentImpl environment = workspace.getConfig().getEnvironments().get(workspace.getConfig().getDefaultEnv()); - assertNotNull(environment); - - final WorkspaceRuntimeImpl runtime = new WorkspaceRuntimeImpl(workspace.getConfig().getDefaultEnv()); - final MachineImpl machine1 = spy(createMachine(workspace.getId(), workspace.getConfig().getDefaultEnv(), true)); - final MachineImpl machine2 = spy(createMachine(workspace.getId(), workspace.getConfig().getDefaultEnv(), false)); - final Map machines = new HashMap<>(); + MachineImpl machine1 = spy(createMachine(workspace.getId(), workspace.getConfig().getDefaultEnv(), true)); + MachineImpl machine2 = spy(createMachine(workspace.getId(), workspace.getConfig().getDefaultEnv(), false)); + Map machines = new HashMap<>(); machines.put(machine1.getId(), machine1); machines.put(machine2.getId(), machine2); - runtime.getMachines().addAll(machines.values()); - runtime.setDevMachine(machine1); - - final RuntimeDescriptor descriptor = mock(RuntimeDescriptor.class); - when(descriptor.getRuntimeStatus()).thenReturn(status); - when(descriptor.getRuntime()).thenReturn(runtime); - return descriptor; + WorkspaceRuntimeImpl runtime = new WorkspaceRuntimeImpl(workspace.getConfig().getDefaultEnv(), machines.values()); + doAnswer(inv -> { + workspace.setStatus(status); + workspace.setRuntime(runtime); + return null; + }).when(runtimes).injectRuntime(workspace); + return runtime; } private WorkspaceImpl createAndMockWorkspace() throws NotFoundException, ServerException { @@ -950,6 +892,16 @@ private WorkspaceImpl createAndMockWorkspace(WorkspaceConfig cfg, String namespa return workspace; } + private void mockStart(Workspace workspace) throws Exception { + CompletableFuture cmpFuture = CompletableFuture.completedFuture(mock(WorkspaceRuntimeImpl.class)); + when(runtimes.startAsync(eq(workspace), anyString(), anyBoolean())).thenReturn(cmpFuture); + } + + private void mockAnyWorkspaceStart() throws Exception { + CompletableFuture cmpFuture = CompletableFuture.completedFuture(mock(WorkspaceRuntimeImpl.class)); + when(runtimes.startAsync(anyObject(), anyString(), anyBoolean())).thenReturn(cmpFuture); + } + private static WorkspaceConfigImpl createConfig() { EnvironmentImpl environment = new EnvironmentImpl(new EnvironmentRecipeImpl("type", "contentType", diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimeIntegrationTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimeIntegrationTest.java index 67465bb61d7..ae02d54d039 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimeIntegrationTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimeIntegrationTest.java @@ -15,7 +15,6 @@ import org.eclipse.che.api.agent.server.AgentRegistry; import org.eclipse.che.api.agent.server.impl.AgentSorter; import org.eclipse.che.api.agent.server.launcher.AgentLauncherFactory; -import org.eclipse.che.api.core.ApiException; import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.model.workspace.Environment; import org.eclipse.che.api.core.notification.EventService; @@ -33,6 +32,7 @@ import org.eclipse.che.api.machine.server.spi.Instance; import org.eclipse.che.api.machine.server.spi.SnapshotDao; import org.eclipse.che.api.machine.server.util.RecipeDownloader; +import org.eclipse.che.api.workspace.server.model.impl.WorkspaceRuntimeImpl; import org.eclipse.che.api.workspace.shared.dto.EnvironmentDto; import org.eclipse.che.api.workspace.shared.dto.ExtendedMachineDto; import org.eclipse.che.api.workspace.shared.dto.WorkspaceConfigDto; @@ -40,6 +40,8 @@ import org.eclipse.che.commons.env.EnvironmentContext; import org.eclipse.che.commons.subject.SubjectImpl; import org.eclipse.che.commons.test.mockito.answer.WaitingAnswer; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.testng.MockitoTestNGListener; import org.slf4j.Logger; @@ -48,6 +50,7 @@ import org.testng.annotations.Listeners; import org.testng.annotations.Test; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -75,29 +78,31 @@ public class WorkspaceRuntimeIntegrationTest { private static final String ENV_NAME = "default-env"; @Mock - private EventService eventService; + private EventService eventService; @Mock - private MachineInstanceProviders machineInstanceProviders; + private MachineInstanceProviders machineInstanceProviders; @Mock - private EnvironmentParser environmentParser; + private EnvironmentParser environmentParser; @Mock - private MachineInstanceProvider instanceProvider; + private MachineInstanceProvider instanceProvider; @Mock - private InfrastructureProvisioner infrastructureProvisioner; + private InfrastructureProvisioner infrastructureProvisioner; @Mock - private RecipeDownloader recipeDownloader; + private RecipeDownloader recipeDownloader; @Mock - private ContainerNameGenerator containerNameGenerator; + private ContainerNameGenerator containerNameGenerator; @Mock - private AgentRegistry agentRegistry; + private AgentRegistry agentRegistry; @Mock - private AgentSorter agentSorter; + private AgentSorter agentSorter; @Mock - private AgentLauncherFactory launcherFactory; + private AgentLauncherFactory launcherFactory; @Mock - private WorkspaceSharedPool sharedPool; + private WorkspaceSharedPool sharedPool; @Mock - private SnapshotDao snapshotDao; + private SnapshotDao snapshotDao; + @Captor + private ArgumentCaptor> taskCaptor; private ExecutorService executor; private WorkspaceRuntimes runtimes; @@ -141,7 +146,7 @@ public void tearDown() throws Exception { // Check for https://github.com/codenvy/codenvy/issues/593 @Test(expectedExceptions = NotFoundException.class, - expectedExceptionsMessageRegExp = "Workspace with id '" + WORKSPACE_ID + "' is not running.") + expectedExceptionsMessageRegExp = "Workspace with id '" + WORKSPACE_ID + "' is not running") public void environmentEngineShouldDestroyAllMachinesBeforeRemovalOfEnvironmentRecord() throws Exception { // given EnvironmentDto environment = newDto(EnvironmentDto.class); @@ -176,7 +181,9 @@ public void environmentEngineShouldDestroyAllMachinesBeforeRemovalOfEnvironmentR any(LineConsumer.class))) .thenReturn(instance); - runtimes.start(workspace, ENV_NAME, false); + runtimes.startAsync(workspace, ENV_NAME, false); + verify(sharedPool).submit(taskCaptor.capture()); + taskCaptor.getValue().call(); WaitingAnswer waitingAnswer = new WaitingAnswer<>(); doAnswer(waitingAnswer).when(instance).destroy(); @@ -185,7 +192,7 @@ public void environmentEngineShouldDestroyAllMachinesBeforeRemovalOfEnvironmentR executor.execute(() -> { try { runtimes.stop(WORKSPACE_ID); - } catch (ApiException e) { + } catch (Exception e) { LOG.error(e.getLocalizedMessage(), e); } }); @@ -194,7 +201,7 @@ public void environmentEngineShouldDestroyAllMachinesBeforeRemovalOfEnvironmentR // then // no exception - environment and workspace are still running - runtimes.get(WORKSPACE_ID); + runtimes.getRuntime(WORKSPACE_ID); // let instance removal proceed waitingAnswer.completeAnswer(); // verify destroying was called @@ -203,6 +210,6 @@ public void environmentEngineShouldDestroyAllMachinesBeforeRemovalOfEnvironmentR // wait to ensure that removal of runtime is finished Thread.sleep(500); // runtime is removed - now getting of it should throw an exception - runtimes.get(WORKSPACE_ID); + runtimes.getRuntime(WORKSPACE_ID); } } diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java index 60b9a73bb26..9a922eb155d 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceRuntimesTest.java @@ -10,20 +10,28 @@ *******************************************************************************/ package org.eclipse.che.api.workspace.server; -import org.eclipse.che.account.spi.AccountImpl; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Futures; + import org.eclipse.che.api.agent.server.AgentRegistry; import org.eclipse.che.api.agent.server.impl.AgentSorter; import org.eclipse.che.api.agent.server.launcher.AgentLauncherFactory; import org.eclipse.che.api.core.ConflictException; import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.ServerException; -import org.eclipse.che.api.core.model.machine.Machine; import org.eclipse.che.api.core.model.machine.MachineConfig; +import org.eclipse.che.api.core.model.machine.MachineStatus; import org.eclipse.che.api.core.model.workspace.Environment; +import org.eclipse.che.api.core.model.workspace.ExtendedMachine; +import org.eclipse.che.api.core.model.workspace.Workspace; import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; import org.eclipse.che.api.core.notification.EventService; import org.eclipse.che.api.environment.server.CheEnvironmentEngine; import org.eclipse.che.api.environment.server.NoOpMachineInstance; +import org.eclipse.che.api.environment.server.exception.EnvironmentException; +import org.eclipse.che.api.environment.server.exception.EnvironmentNotRunningException; +import org.eclipse.che.api.environment.server.exception.EnvironmentStartInterruptedException; import org.eclipse.che.api.machine.server.exception.SnapshotException; import org.eclipse.che.api.machine.server.model.impl.MachineConfigImpl; import org.eclipse.che.api.machine.server.model.impl.MachineImpl; @@ -33,52 +41,64 @@ import org.eclipse.che.api.machine.server.model.impl.SnapshotImpl; import org.eclipse.che.api.machine.server.spi.Instance; import org.eclipse.che.api.machine.server.spi.SnapshotDao; -import org.eclipse.che.api.workspace.server.WorkspaceRuntimes.RuntimeDescriptor; +import org.eclipse.che.api.workspace.server.WorkspaceRuntimes.RuntimeState; import org.eclipse.che.api.workspace.server.model.impl.EnvironmentImpl; +import org.eclipse.che.api.workspace.server.model.impl.ExtendedMachineImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceConfigImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceImpl; import org.eclipse.che.api.workspace.server.model.impl.WorkspaceRuntimeImpl; import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent.EventType; -import org.eclipse.che.commons.lang.NameGenerator; import org.eclipse.che.dto.server.DtoFactory; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.mockito.testng.MockitoTestNGListener; import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; import org.testng.annotations.Listeners; import org.testng.annotations.Test; -import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.Callable; - -import static java.util.Arrays.asList; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static java.lang.String.format; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; -import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.RUNNING; -import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.STOPPED; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; /** * @author Yevhenii Voevodin @@ -87,395 +107,514 @@ @Listeners(MockitoTestNGListener.class) public class WorkspaceRuntimesTest { - private static final String WORKSPACE_ID = "workspace123"; - private static final String ENV_NAME = "default-env"; - @Mock - private EventService eventService; + private EventService eventService; @Mock - private CheEnvironmentEngine envEngine; + private CheEnvironmentEngine envEngine; @Mock - private AgentSorter agentSorter; + private AgentSorter agentSorter; @Mock - private AgentLauncherFactory launcherFactory; + private AgentLauncherFactory launcherFactory; @Mock - private AgentRegistry agentRegistry; + private AgentRegistry agentRegistry; @Mock - private WorkspaceSharedPool sharedPool; + private WorkspaceSharedPool sharedPool; @Mock - private SnapshotDao snapshotDao; + private SnapshotDao snapshotDao; + @Mock + private Future runtimeFuture; + @Mock + private WorkspaceRuntimes.StartTask startTask; @Captor private ArgumentCaptor eventCaptor; @Captor - private ArgumentCaptor taskCaptor; + private ArgumentCaptor> taskCaptor; @Captor private ArgumentCaptor> snapshotsCaptor; - private WorkspaceRuntimes runtimes; + private WorkspaceRuntimes runtimes; + private ConcurrentMap runtimeStates; @BeforeMethod - public void setUp(Method method) throws Exception { - runtimes = spy(new WorkspaceRuntimes(eventService, - envEngine, - agentSorter, - launcherFactory, - agentRegistry, - snapshotDao, - sharedPool)); - - List machines = asList(createMachine(true), createMachine(false)); - when(envEngine.start(anyString(), - anyString(), - any(Environment.class), - anyBoolean(), - any())) - .thenReturn(machines); - when(envEngine.getMachines(WORKSPACE_ID)).thenReturn(machines); + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + runtimes = new WorkspaceRuntimes(eventService, + envEngine, + agentSorter, + launcherFactory, + agentRegistry, + snapshotDao, + sharedPool, + runtimeStates = new ConcurrentHashMap<>()); + } + + @Test(dataProvider = "allStatuses") + public void getsStatus(WorkspaceStatus status) throws Exception { + setRuntime("workspace", status); + + assertEquals(runtimes.getStatus("workspace"), status); } @Test(expectedExceptions = NotFoundException.class, - expectedExceptionsMessageRegExp = "Workspace with id '.*' is not running.") - public void shouldThrowNotFoundExceptionIfWorkspaceRuntimeDoesNotExist() throws Exception { - runtimes.get(WORKSPACE_ID); + expectedExceptionsMessageRegExp = "Workspace with id 'non_running' is not running") + public void throwsNotFoundExceptionWhenGettingNonExistingRuntime() throws Exception { + runtimes.getRuntime("non_running"); } - @Test(expectedExceptions = ServerException.class, - expectedExceptionsMessageRegExp = "Dev machine is not found in active environment of workspace 'workspace123'") - public void shouldThrowExceptionOnGetRuntimesIfDevMachineIsMissingInTheEnvironment() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + @Test + public void returnsStoppedStatusWhenWorkspaceIsNotRunning() throws Exception { + assertEquals(runtimes.getStatus("not_running"), WorkspaceStatus.STOPPED); + } - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - when(envEngine.getMachines(workspace.getId())) - .thenReturn(asList(createMachine(false), createMachine(false))); + @Test + public void getsRuntime() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name"); + List machines = prepareMachines("workspace", "env-name"); - // when - runtimes.get(workspace.getId()); + assertEquals(runtimes.getRuntime("workspace"), new WorkspaceRuntimeImpl("env-name", machines)); + verify(envEngine).getMachines("workspace"); } @Test - public void shouldFetchMachinesFromEnvEngineOnGetRuntime() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - Instance devMachine = createMachine(true); - List machines = asList(devMachine, createMachine(false)); - when(envEngine.start(anyString(), - anyString(), - any(Environment.class), - anyBoolean(), - any())) - .thenReturn(machines); - when(envEngine.getMachines(WORKSPACE_ID)).thenReturn(machines); + public void hasRuntime() { + setRuntime("workspace", WorkspaceStatus.STARTING); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + assertTrue(runtimes.hasRuntime("workspace")); + } - // when - RuntimeDescriptor runtimeDescriptor = runtimes.get(workspace.getId()); + @Test + public void doesNotHaveRuntime() { + assertFalse(runtimes.hasRuntime("not_running")); + } - // then - RuntimeDescriptor expected = new RuntimeDescriptor(WorkspaceStatus.RUNNING, - new WorkspaceRuntimeImpl(workspace.getConfig() - .getDefaultEnv(), - devMachine.getRuntime() - .projectsRoot(), - machines, - devMachine)); - verify(envEngine, times(2)).getMachines(workspace.getId()); - assertEquals(runtimeDescriptor, expected); - } - - @Test(expectedExceptions = ServerException.class, - expectedExceptionsMessageRegExp = "Could not perform operation because application server is stopping") - public void shouldNotStartTheWorkspaceIfPostConstructWasIsInvoked() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.cleanup(); + @Test + public void injectsRuntime() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name"); + List machines = prepareMachines("workspace", "env-name"); + WorkspaceImpl workspace = WorkspaceImpl.builder() + .setId("workspace") + .build(); - // when - runtimes.start(createWorkspace(), workspace.getConfig().getDefaultEnv(), false); + runtimes.injectRuntime(workspace); + + assertEquals(workspace.getStatus(), WorkspaceStatus.RUNNING); + assertEquals(workspace.getRuntime(), new WorkspaceRuntimeImpl("env-name", machines)); } @Test - public void workspaceShouldNotHaveRuntimeIfEnvStartFails() throws Exception { - // given - when(envEngine.start(anyString(), - anyString(), - any(Environment.class), - anyBoolean(), - any())) - .thenThrow(new ServerException("Test env start error")); - WorkspaceImpl workspaceMock = createWorkspace(); + public void injectsStoppedStatusWhenWorkspaceDoesNotHaveRuntime() throws Exception { + WorkspaceImpl workspace = WorkspaceImpl.builder() + .setId("workspace") + .build(); - try { - // when - runtimes.start(workspaceMock, - workspaceMock.getConfig().getDefaultEnv(), - false); - } catch (Exception ex) { - // then - assertFalse(runtimes.hasRuntime(workspaceMock.getId())); - } + runtimes.injectRuntime(workspace); + + assertEquals(workspace.getStatus(), WorkspaceStatus.STOPPED); + assertNull(workspace.getRuntime()); } @Test - public void workspaceShouldContainAllMachinesAndBeInRunningStatusAfterSuccessfulStart() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + public void injectsStatusAndEmptyMachinesWhenCanNotGetEnvironmentMachines() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name"); + setNoMachinesForWorkspace("workspace"); + WorkspaceImpl workspace = WorkspaceImpl.builder() + .setId("workspace") + .build(); - // when - RuntimeDescriptor runningWorkspace = runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + runtimes.injectRuntime(workspace); - // then - assertEquals(runningWorkspace.getRuntimeStatus(), RUNNING); - assertNotNull(runningWorkspace.getRuntime().getDevMachine()); - assertEquals(runningWorkspace.getRuntime().getMachines().size(), 2); + assertEquals(workspace.getStatus(), WorkspaceStatus.RUNNING); + assertEquals(workspace.getRuntime().getActiveEnv(), "env-name"); + assertTrue(workspace.getRuntime().getMachines().isEmpty()); + } + + @Test + public void startsWorkspace() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + List machines = allowEnvironmentStart(workspace, "env-name"); + prepareMachines(workspace.getId(), machines); + + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + captureAsyncTaskAndExecuteSynchronously(); + WorkspaceRuntimeImpl runtime = cmpFuture.get(); + + assertEquals(runtimes.getStatus(workspace.getId()), WorkspaceStatus.RUNNING); + assertEquals(runtime.getActiveEnv(), "env-name"); + assertEquals(runtime.getMachines().size(), machines.size()); + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.RUNNING, + EventType.RUNNING, + null)); } @Test(expectedExceptions = ConflictException.class, - expectedExceptionsMessageRegExp = "Could not start workspace '.*' because its status is 'RUNNING'") - public void shouldNotStartWorkspaceIfItIsAlreadyRunning() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + expectedExceptionsMessageRegExp = "Could not start workspace 'test-workspace' because its status is 'RUNNING'") + public void throwsConflictExceptionWhenWorkspaceIsRunning() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + setRuntime(workspace.getId(), WorkspaceStatus.RUNNING); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - // when - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + runtimes.startAsync(workspace, "env-name", false); } @Test - public void testCleanup() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + public void cancelsWorkspaceStartIfEnvironmentStartIsInterrupted() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + rejectEnvironmentStart(workspace, "env-name", new EnvironmentStartInterruptedException(workspace.getId(), "env-name")); - runtimes.cleanup(); + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); - // when, then - assertFalse(runtimes.hasRuntime(workspace.getId())); + captureAndVerifyRuntimeStateAfterInterruption(workspace, cmpFuture); } @Test - public void shouldStopRunningWorkspace() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + public void failsWorkspaceStartWhenEnvironmentStartIsFailed() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + rejectEnvironmentStart(workspace, "env-name", new EnvironmentException("no no no!")); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - // when - runtimes.stop(workspace.getId()); + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); - // then + try { + captureAsyncTaskAndExecuteSynchronously(); + } catch (EnvironmentException x) { + assertEquals(x.getMessage(), "no no no!"); + verifyCompletionException(cmpFuture, EnvironmentException.class, "no no no!"); + } assertFalse(runtimes.hasRuntime(workspace.getId())); + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.STOPPED, + EventType.ERROR, + "Start of environment 'env-name' failed. Error: no no no!")); } - @Test(expectedExceptions = NotFoundException.class, - expectedExceptionsMessageRegExp = "Workspace with id 'workspace123' is not running.") - public void shouldThrowNotFoundExceptionWhenStoppingWorkspaceWhichDoesNotHaveRuntime() throws Exception { - runtimes.stop(WORKSPACE_ID); + @Test + public void interruptsStartAfterEnvironmentIsStartedButRuntimeStatusIsNotRunning() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + // let's say status is changed to STOPPING by stop method, + // but starting thread hasn't been interrupted yet + allowEnvironmentStart(workspace, "env-name", () -> setRuntime("workspace", WorkspaceStatus.STOPPING)); + + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + + captureAndVerifyRuntimeStateAfterInterruption(workspace, cmpFuture); + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.STOPPED, + null)); + verify(envEngine).stop(workspace.getId()); } @Test - public void startedRuntimeShouldBeTheSameToRuntimeTakenFromGetMethod() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + public void interruptsStartAfterEnvironmentIsStartedButThreadIsInterrupted() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + // the status is successfully updated from STARTING -> RUNNING but after + // that thread is interrupted so #stop is waiting for starting thread to stop the environment + allowEnvironmentStart(workspace, "env-name", () -> Thread.currentThread().interrupt()); + + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + + captureAndVerifyRuntimeStateAfterInterruption(workspace, cmpFuture); + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.STOPPED, + null)); + verify(envEngine).stop(workspace.getId()); + } - // when - RuntimeDescriptor descriptorFromStartMethod = runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - RuntimeDescriptor descriptorFromGetMethod = runtimes.get(workspace.getId()); + @Test + public void throwsStartInterruptedExceptionWhenStartIsInterruptedAndEnvironmentStopIsFailed() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + // let's say status is changed to STOPPING by stop method, + // but starting thread hasn't been interrupted yet + allowEnvironmentStart(workspace, "env-name", () -> Thread.currentThread().interrupt()); + rejectEnvironmentStop(workspace, new ServerException("no!")); + + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + + captureAndVerifyRuntimeStateAfterInterruption(workspace, cmpFuture); + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.ERROR, + "no!")); + verify(envEngine).stop(workspace.getId()); + } - // then - assertEquals(descriptorFromStartMethod, - descriptorFromGetMethod); + @Test + public void releasesClientsWhoWaitForStartTaskResultAndTaskIsCompleted() throws Exception { + ExecutorService pool = Executors.newCachedThreadPool(); + CountDownLatch releasedLatch = new CountDownLatch(5); + // this thread + 5 awaiting threads + CyclicBarrier callTaskBarrier = new CyclicBarrier(6); + + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + allowEnvironmentStart(workspace, "env-name"); + + // the action + runtimes.startAsync(workspace, "env-name", false); + + // register waiters + for (int i = 0; i < 5; i++) { + WorkspaceRuntimes.StartTask startTask = runtimeStates.get(workspace.getId()).startTask; + pool.submit(() -> { + // wait all the task to meet this barrier + callTaskBarrier.await(); + + // wait for start task to finish + startTask.await(); + + // good, release a part + releasedLatch.countDown(); + return null; + }); + } + + callTaskBarrier.await(); + captureAsyncTaskAndExecuteSynchronously(); + try { + assertTrue(releasedLatch.await(2, TimeUnit.SECONDS), "start task wait clients are not released"); + } finally { + shutdownAndWaitPool(pool); + } } @Test - public void startingEventShouldBePublishedBeforeStart() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + public void stopsRunningWorkspace() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING); - // when - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + runtimes.stop("workspace"); - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspace.getId()) - .withStatus(WorkspaceStatus.STARTING) - .withEventType(EventType.STARTING) - .withPrevStatus(WorkspaceStatus.STOPPED)); + verify(envEngine).stop("workspace"); + verifyEventsSequence(event("workspace", + WorkspaceStatus.RUNNING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.STOPPED, + null)); + assertFalse(runtimeStates.containsKey("workspace")); } @Test - public void runningEventShouldBePublishedAfterEnvStart() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); + public void stopsTheRunningWorkspaceWhileServerExceptionOccurs() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING); + doThrow(new ServerException("no!")).when(envEngine).stop("workspace"); - // when - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + try { + runtimes.stop("workspace"); + } catch (ServerException x) { + assertEquals(x.getMessage(), "no!"); + } - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.RUNNING) - .withWorkspaceId(workspace.getId()) - .withEventType(EventType.RUNNING) - .withPrevStatus(WorkspaceStatus.STARTING)); + verify(envEngine).stop("workspace"); + assertFalse(runtimeStates.containsKey("workspace")); + verifyEventsSequence(event("workspace", + WorkspaceStatus.RUNNING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.ERROR, + "no!")); } @Test - public void errorEventShouldBePublishedIfDevMachineFailedToStart() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - when(envEngine.start(anyString(), - anyString(), - any(Environment.class), - anyBoolean(), - any())) - .thenReturn(singletonList(createMachine(false))); + public void stopsTheRunningWorkspaceAndRethrowsTheErrorDifferentFromServerException() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING); + doThrow(new EnvironmentNotRunningException("no!")).when(envEngine).stop("workspace"); try { - // when - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - - } catch (Exception e) { - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspace.getId()) - .withEventType(EventType.ERROR) - .withPrevStatus(WorkspaceStatus.STARTING)); + runtimes.stop("workspace"); + } catch (ServerException x) { + assertEquals(x.getMessage(), "no!"); + assertTrue(x.getCause() instanceof EnvironmentNotRunningException); } + + verify(envEngine).stop("workspace"); + assertFalse(runtimeStates.containsKey("workspace")); + verifyEventsSequence(event("workspace", + WorkspaceStatus.RUNNING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.ERROR, + "no!")); } @Test - public void stoppingEventShouldBePublishedBeforeStop() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + public void cancellationOfPendingStartTask() throws Throwable { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + when(sharedPool.submit(any())).thenReturn(Futures.immediateFuture(null)); - // when + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + + // the real start is not being executed, fake sharedPool suppressed it + // so the situation is the same to the one if the task is cancelled before + // executor service started executing it runtimes.stop(workspace.getId()); - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.STOPPING) - .withWorkspaceId(workspace.getId()) - .withEventType(EventType.STOPPING) - .withPrevStatus(WorkspaceStatus.RUNNING)); + // start awaiting clients MUST receive interruption + try { + cmpFuture.get(); + } catch (ExecutionException x) { + verifyCompletionException(cmpFuture, + EnvironmentStartInterruptedException.class, + "Start of environment 'env-name' in workspace 'workspace' is interrupted"); + } + + // if there is a state when the future is being cancelled, + // and start task is marked as used, executor must not execute the + // task but throw cancellation exception instead, once start task is + // completed clients receive interrupted exception and cancellation doesn't bother them + try { + captureAsyncTaskAndExecuteSynchronously(); + } catch (CancellationException cancelled) { + assertEquals(cancelled.getMessage(), "Start of the workspace 'workspace' was cancelled"); + } + + verifyEventsSequence(event("workspace", + WorkspaceStatus.STOPPED, + WorkspaceStatus.STARTING, + EventType.STARTING, + null), + event("workspace", + WorkspaceStatus.STARTING, + WorkspaceStatus.STOPPING, + EventType.STOPPING, + null), + event("workspace", + WorkspaceStatus.STOPPING, + WorkspaceStatus.STOPPED, + EventType.STOPPED, + null)); } @Test - public void stoppedEventShouldBePublishedAfterEnvStop() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + public void cancellationOfRunningStartTask() throws Exception { + setRuntime("workspace", + WorkspaceStatus.STARTING, + "env-name", + runtimeFuture, + startTask); + doThrow(new EnvironmentStartInterruptedException("workspace", "env-name")).when(startTask).await(); - // when - runtimes.stop(workspace.getId()); + runtimes.stop("workspace"); - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.STOPPED) - .withWorkspaceId(workspace.getId()) - .withEventType(EventType.STOPPED) - .withPrevStatus(WorkspaceStatus.STOPPING)); + verify(runtimeFuture).cancel(true); + verify(startTask).await(); + } + + @Test(expectedExceptions = NotFoundException.class, + expectedExceptionsMessageRegExp = "Workspace with id 'workspace' is not running") + public void throwsNotFoundExceptionWhenStoppingNotRunningWorkspace() throws Exception { + runtimes.stop("workspace"); + } + + @Test(expectedExceptions = ConflictException.class, + expectedExceptionsMessageRegExp = "Couldn't stop the workspace 'workspace' because its status is '.*'.*", + dataProvider = "notAllowedToStopStatuses") + public void doesNotStopTheWorkspaceWhenStatusIsWrong(WorkspaceStatus status) throws Exception { + setRuntime("workspace", status); + + runtimes.stop("workspace"); } @Test - public void errorEventShouldBePublishedIfEnvFailedToStop() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + public void cleanup() throws Exception { + setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name"); - try { - // when - runtimes.stop(workspace.getId()); - } catch (Exception e) { - // then - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspace.getId()) - .withEventType(EventType.ERROR) - .withPrevStatus(WorkspaceStatus.STOPPING) - .withError("Test error")); - } + runtimes.cleanup(); + + assertFalse(runtimes.hasRuntime("workspace")); + verify(envEngine).stop("workspace"); } @Test - public void shouldBeAbleToStartMachine() throws Exception { - // when - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - MachineConfigImpl config = createConfig(false); - Instance instance = mock(Instance.class); - when(envEngine.startMachine(anyString(), any(MachineConfig.class), any())).thenReturn(instance); - when(instance.getConfig()).thenReturn(config); + public void startedRuntimeAndReturnedFromGetMethodAreTheSame() throws Exception { + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + allowEnvironmentStart(workspace, "env-name"); + prepareMachines(workspace.getId(), "env-name"); - // when - Instance actual = runtimes.startMachine(workspace.getId(), config); + CompletableFuture cmpFuture = runtimes.startAsync(workspace, "env-name", false); + captureAsyncTaskAndExecuteSynchronously(); - // then - assertEquals(actual, instance); - verify(envEngine).startMachine(eq(workspace.getId()), eq(config), any()); + assertEquals(cmpFuture.get(), runtimes.getRuntime(workspace.getId())); } @Test - public void shouldAddTerminalAgentOnMachineStart() throws Exception { + public void shouldBeAbleToStartMachine() throws Exception { // when - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - MachineConfigImpl config = createConfig(false); + setRuntime("workspace", WorkspaceStatus.RUNNING, "env-name"); + MachineConfig config = newMachine("workspace", "env-name", "new", false).getConfig(); Instance instance = mock(Instance.class); when(envEngine.startMachine(anyString(), any(MachineConfig.class), any())).thenReturn(instance); when(instance.getConfig()).thenReturn(config); // when - Instance actual = runtimes.startMachine(workspace.getId(), config); + Instance actual = runtimes.startMachine("workspace", config); // then assertEquals(actual, instance); - verify(envEngine).startMachine(eq(workspace.getId()), - eq(config), - eq(singletonList("org.eclipse.che.terminal"))); - verify(runtimes).launchAgents(instance, singletonList("org.eclipse.che.terminal")); + verify(envEngine).startMachine(eq("workspace"), eq(config), any()); } @Test(expectedExceptions = NotFoundException.class, expectedExceptionsMessageRegExp = "Workspace with id '.*' is not running") public void shouldNotStartMachineIfEnvironmentIsNotRunning() throws Exception { // when - MachineConfigImpl config = createConfig(false); - - // when - runtimes.startMachine("someWsID", config); + runtimes.startMachine("someWsID", mock(MachineConfig.class)); // then verify(envEngine, never()).startMachine(anyString(), any(MachineConfig.class), any()); @@ -484,16 +623,13 @@ public void shouldNotStartMachineIfEnvironmentIsNotRunning() throws Exception { @Test public void shouldBeAbleToStopMachine() throws Exception { // when - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); + setRuntime("workspace", WorkspaceStatus.RUNNING); // when - runtimes.stopMachine(workspace.getId(), "testMachineId"); + runtimes.stopMachine("workspace", "testMachineId"); // then - verify(envEngine).stopMachine(workspace.getId(), "testMachineId"); + verify(envEngine).stopMachine("workspace", "testMachineId"); } @Test(expectedExceptions = NotFoundException.class, @@ -509,112 +645,50 @@ public void shouldNotStopMachineIfEnvironmentIsNotRunning() throws Exception { @Test public void shouldBeAbleToGetMachine() throws Exception { // given - Instance expected = createMachine(false); - when(envEngine.getMachine(WORKSPACE_ID, expected.getId())).thenReturn(expected); + Instance expected = newMachine("workspace", "env-name", "existing", false); + when(envEngine.getMachine("workspace", expected.getId())).thenReturn(expected); // when - Instance actualMachine = runtimes.getMachine(WORKSPACE_ID, expected.getId()); + Instance actualMachine = runtimes.getMachine("workspace", expected.getId()); // then assertEquals(actualMachine, expected); - verify(envEngine).getMachine(WORKSPACE_ID, expected.getId()); - } - - @Test - public void shouldBeAbleToGetStatusOfRunningWorkspace() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - - // when - WorkspaceStatus status = runtimes.getStatus(workspace.getId()); - - // then - assertEquals(status, RUNNING); - } - - - @Test - public void shouldBeAbleToGetStatusOfStoppedWorkspace() throws Exception { - // given - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - runtimes.stop(workspace.getId()); - - // when - WorkspaceStatus status = runtimes.getStatus(workspace.getId()); - - // then - assertEquals(status, STOPPED); + verify(envEngine).getMachine("workspace", expected.getId()); } @Test(expectedExceptions = NotFoundException.class, expectedExceptionsMessageRegExp = "test exception") public void shouldThrowExceptionIfGetMachineFromEnvEngineThrowsException() throws Exception { // given - Instance expected = createMachine(false); - when(envEngine.getMachine(WORKSPACE_ID, expected.getId())) + Instance expected = newMachine("workspace", "env-name", "existing", false); + when(envEngine.getMachine("workspace", expected.getId())) .thenThrow(new NotFoundException("test exception")); // when - runtimes.getMachine(WORKSPACE_ID, expected.getId()); - - // then - verify(envEngine).getMachine(WORKSPACE_ID, expected.getId()); - } - - @Test - public void shouldBeAbleToGetAllWorkspacesWithExistingRuntime() throws Exception { - // then - Map expectedWorkspaces = new HashMap<>(); - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, - workspace.getConfig().getDefaultEnv(), - false); - expectedWorkspaces.put(workspace.getId(), - new WorkspaceRuntimes.WorkspaceState(RUNNING, - workspace.getConfig().getDefaultEnv())); - WorkspaceImpl workspace2 = spy(createWorkspace()); - when(workspace2.getId()).thenReturn("testWsId"); - when(envEngine.getMachines(workspace2.getId())) - .thenReturn(Collections.singletonList(createMachine(true))); - runtimes.start(workspace2, - workspace2.getConfig().getDefaultEnv(), - false); - expectedWorkspaces.put(workspace2.getId(), - new WorkspaceRuntimes.WorkspaceState(RUNNING, - workspace2.getConfig().getDefaultEnv())); - - // when - Map actualWorkspaces = runtimes.getWorkspaces(); + runtimes.getMachine("workspace", expected.getId()); // then - assertEquals(actualWorkspaces, expectedWorkspaces); + verify(envEngine).getMachine("workspace", expected.getId()); } @Test public void changesStatusFromRunningToSnapshotting() throws Exception { - final WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + setRuntime("workspace", WorkspaceStatus.RUNNING); - runtimes.snapshotAsync(workspace.getId()); + runtimes.snapshotAsync("workspace"); - assertEquals(runtimes.get(workspace.getId()).getRuntimeStatus(), WorkspaceStatus.SNAPSHOTTING); + assertEquals(runtimes.getStatus("workspace"), WorkspaceStatus.SNAPSHOTTING); } @Test public void changesStatusFromSnapshottingToRunning() throws Exception { - final WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + setRuntime(workspace.getId(), WorkspaceStatus.RUNNING, "env-name"); runtimes.snapshotAsync(workspace.getId()); captureAsyncTaskAndExecuteSynchronously(); - assertEquals(runtimes.get(workspace.getId()).getRuntimeStatus(), WorkspaceStatus.RUNNING); + assertEquals(runtimes.getStatus(workspace.getId()), WorkspaceStatus.RUNNING); } @Test(expectedExceptions = NotFoundException.class, @@ -626,41 +700,40 @@ public void throwsNotFoundExceptionWhenBeginningSnapshottingForNonExistingWorksp @Test(expectedExceptions = ConflictException.class, expectedExceptionsMessageRegExp = "Workspace with id '.*' is not 'RUNNING', it's status is 'SNAPSHOTTING'") public void throwsConflictExceptionWhenBeginningSnapshottingForNotRunningWorkspace() throws Exception { - final WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + setRuntime("workspace", WorkspaceStatus.RUNNING); - runtimes.snapshotAsync(workspace.getId()); - runtimes.snapshotAsync(workspace.getId()); + runtimes.snapshotAsync("workspace"); + runtimes.snapshotAsync("workspace"); } @Test(expectedExceptions = ServerException.class, expectedExceptionsMessageRegExp = "can't save") public void failsToCreateSnapshotWhenDevMachineSnapshottingFailed() throws Exception { - final WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + setRuntime(workspace.getId(), WorkspaceStatus.RUNNING); + prepareMachines(workspace.getId(), "env-name"); when(envEngine.saveSnapshot(any(), any())).thenThrow(new ServerException("can't save")); try { runtimes.snapshot(workspace.getId()); } catch (Exception x) { - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspace.getId()) - .withStatus(WorkspaceStatus.SNAPSHOTTING) - .withPrevStatus(WorkspaceStatus.RUNNING) - .withEventType(EventType.SNAPSHOT_CREATING)); - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withWorkspaceId(workspace.getId()) - .withError("can't save") - .withStatus(WorkspaceStatus.RUNNING) - .withPrevStatus(WorkspaceStatus.SNAPSHOTTING) - .withEventType(EventType.SNAPSHOT_CREATION_ERROR)); + verifyEventsSequence(event(workspace.getId(), + WorkspaceStatus.RUNNING, + WorkspaceStatus.SNAPSHOTTING, + EventType.SNAPSHOT_CREATING, + null), + event(workspace.getId(), + WorkspaceStatus.SNAPSHOTTING, + WorkspaceStatus.RUNNING, + EventType.SNAPSHOT_CREATION_ERROR, + "can't save")); throw x; } } - @Test(expectedExceptions = ServerException.class, expectedExceptionsMessageRegExp = "test") + @Test public void removesNewlyCreatedSnapshotsWhenFailedToSaveTheirsMetadata() throws Exception { - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + setRuntime(workspace.getId(), WorkspaceStatus.RUNNING, "env-name"); doThrow(new SnapshotException("test")).when(snapshotDao) .replaceSnapshots(any(), any(), any()); SnapshotImpl snapshot = mock(SnapshotImpl.class); @@ -668,101 +741,283 @@ public void removesNewlyCreatedSnapshotsWhenFailedToSaveTheirsMetadata() throws try { runtimes.snapshot(workspace.getId()); - } catch (Exception x) { - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.SNAPSHOTTING) - .withEventType(EventType.SNAPSHOT_CREATING) - .withPrevStatus(WorkspaceStatus.RUNNING) - .withWorkspaceId(workspace.getId())); - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.RUNNING) - .withEventType(EventType.SNAPSHOT_CREATION_ERROR) - .withWorkspaceId(workspace.getId()) - .withPrevStatus(WorkspaceStatus.SNAPSHOTTING) - .withError("test")); - verify(snapshotDao).replaceSnapshots(any(), - any(), - snapshotsCaptor.capture()); - verify(envEngine, times(snapshotsCaptor.getValue().size())).removeSnapshot(snapshot); - throw x; + } catch (ServerException x) { + assertEquals(x.getMessage(), "test"); } + + verify(snapshotDao).replaceSnapshots(any(), any(), snapshotsCaptor.capture()); + verify(envEngine, times(snapshotsCaptor.getValue().size())).removeSnapshot(snapshot); + verifyEventsSequence(event(workspace.getId(), + WorkspaceStatus.RUNNING, + WorkspaceStatus.SNAPSHOTTING, + EventType.SNAPSHOT_CREATING, + null), + event(workspace.getId(), + WorkspaceStatus.SNAPSHOTTING, + WorkspaceStatus.RUNNING, + EventType.SNAPSHOT_CREATION_ERROR, + "test")); } @Test public void removesOldSnapshotsWhenNewSnapshotsMetadataSuccessfullySaved() throws Exception { - WorkspaceImpl workspace = createWorkspace(); - runtimes.start(workspace, workspace.getConfig().getDefaultEnv(), false); + WorkspaceImpl workspace = newWorkspace("workspace", "env-name"); + setRuntime(workspace.getId(), WorkspaceStatus.RUNNING); SnapshotImpl oldSnapshot = mock(SnapshotImpl.class); doReturn((singletonList(oldSnapshot))).when(snapshotDao) .replaceSnapshots(any(), any(), any()); runtimes.snapshot(workspace.getId()); - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.SNAPSHOTTING) - .withEventType(EventType.SNAPSHOT_CREATING) - .withPrevStatus(WorkspaceStatus.RUNNING) - .withWorkspaceId(workspace.getId())); - verify(eventService).publish(DtoFactory.newDto(WorkspaceStatusEvent.class) - .withStatus(WorkspaceStatus.RUNNING) - .withEventType(EventType.SNAPSHOT_CREATED) - .withPrevStatus(WorkspaceStatus.SNAPSHOTTING) - .withWorkspaceId(workspace.getId())); verify(envEngine).removeSnapshot(oldSnapshot); + verifyEventsSequence(event(workspace.getId(), + WorkspaceStatus.RUNNING, + WorkspaceStatus.SNAPSHOTTING, + EventType.SNAPSHOT_CREATING, + null), + event(workspace.getId(), + WorkspaceStatus.SNAPSHOTTING, + WorkspaceStatus.RUNNING, + EventType.SNAPSHOT_CREATED, + null)); } - private static Instance createMachine(boolean isDev) { - return createMachine(createConfig(isDev)); + @Test + public void getsRuntimesIds() { + setRuntime("workspace1", WorkspaceStatus.STARTING); + setRuntime("workspace2", WorkspaceStatus.RUNNING); + setRuntime("workspace3", WorkspaceStatus.STOPPING); + setRuntime("workspace4", WorkspaceStatus.SNAPSHOTTING); + + assertEquals(runtimes.getRuntimesIds(), Sets.newHashSet("workspace1", + "workspace2", + "workspace3", + "workspace4")); } - private static Instance createMachine(MachineConfig cfg) { - return new TestMachineInstance(MachineImpl.builder() - .setId(NameGenerator.generate("machine", 10)) - .setWorkspaceId(WORKSPACE_ID) - .setEnvName(ENV_NAME) - .setConfig(new MachineConfigImpl(cfg)) - .build()); + private void captureAsyncTaskAndExecuteSynchronously() throws Exception { + verify(sharedPool).submit(taskCaptor.capture()); + taskCaptor.getValue().call(); } - private static MachineConfigImpl createConfig(boolean isDev) { - return MachineConfigImpl.builder() - .setDev(isDev) - .setType("docker") - .setLimits(new MachineLimitsImpl(1024)) - .setSource(new MachineSourceImpl("git").setLocation("location")) - .setName(UUID.randomUUID().toString()) - .build(); + + private void captureAndVerifyRuntimeStateAfterInterruption(Workspace workspace, + CompletableFuture cmpFuture) throws Exception { + try { + captureAsyncTaskAndExecuteSynchronously(); + } catch (EnvironmentStartInterruptedException x) { + String expectedMessage = "Start of environment 'env-name' in workspace 'workspace' is interrupted"; + assertEquals(x.getMessage(), expectedMessage); + verifyCompletionException(cmpFuture, EnvironmentStartInterruptedException.class, expectedMessage); + } + assertFalse(runtimes.hasRuntime(workspace.getId())); } - private static WorkspaceImpl createWorkspace() { - EnvironmentImpl environment = new EnvironmentImpl(null, - null); - WorkspaceConfigImpl wsConfig = WorkspaceConfigImpl.builder() - .setName("test workspace") - .setEnvironments(singletonMap(ENV_NAME, environment)) - .setDefaultEnv(ENV_NAME) - .build(); - return new WorkspaceImpl(WORKSPACE_ID, new AccountImpl("accountId", "user123", "test"), wsConfig); + private void verifyCompletionException(Future f, Class expectedEx, String expectedMessage) { + assertTrue(f.isDone()); + try { + f.get(); + } catch (ExecutionException execEx) { + if (expectedEx.isInstance(execEx.getCause())) { + assertEquals(execEx.getCause().getMessage(), expectedMessage); + } else { + fail(execEx.getMessage(), execEx); + } + } catch (InterruptedException interruptedEx) { + fail(interruptedEx.getMessage(), interruptedEx); + } } - @SuppressWarnings("unchecked") - private void captureAsyncTaskAndExecuteSynchronously() throws Exception { - verify(sharedPool).submit(taskCaptor.capture()); - taskCaptor.getValue().call(); + private void verifyEventsSequence(WorkspaceStatusEvent... expected) { + Iterator it = captureEvents().iterator(); + for (WorkspaceStatusEvent expEvent : expected) { + if (!it.hasNext()) { + fail(format("It is expected to receive the status changed event '%s' -> '%s' " + + "but there are no more events published", + expEvent.getPrevStatus(), + expEvent.getStatus())); + } + WorkspaceStatusEvent cur = it.next(); + if (cur.getPrevStatus() != expEvent.getPrevStatus() || cur.getStatus() != expEvent.getStatus()) { + fail(format("Expected to receive status change '%s' -> '%s', while received '%s' -> '%s'", + expEvent.getPrevStatus(), + expEvent.getStatus(), + cur.getPrevStatus(), + cur.getStatus())); + } + assertEquals(cur, expEvent); + } + if (it.hasNext()) { + WorkspaceStatusEvent next = it.next(); + fail(format("No more events expected, but received '%s' -> '%s'", + next.getPrevStatus(), + next.getStatus())); + } + } + + private static WorkspaceStatusEvent event(String workspaceId, + WorkspaceStatus prevStatus, + WorkspaceStatus status, + EventType eventType, + String error) { + return DtoFactory.newDto(WorkspaceStatusEvent.class) + .withWorkspaceId(workspaceId) + .withStatus(status) + .withPrevStatus(prevStatus) + .withEventType(eventType) + .withError(error); + } + + private List captureEvents() { + verify(eventService, atLeastOnce()).publish(eventCaptor.capture()); + return eventCaptor.getAllValues(); + } + + private void setRuntime(String workspaceId, WorkspaceStatus status) { + runtimeStates.put(workspaceId, new RuntimeState(status, null, null, null)); + } + + private void setRuntime(String workspaceId, WorkspaceStatus status, String envName) { + runtimeStates.put(workspaceId, new RuntimeState(status, envName, null, null)); } - private static class TestMachineInstance extends NoOpMachineInstance { + private void setRuntime(String workspaceId, + WorkspaceStatus status, + String envName, + Future startFuture, + WorkspaceRuntimes.StartTask startTask) { + runtimeStates.put(workspaceId, new RuntimeState(status, envName, startTask, startFuture)); + } - MachineRuntimeInfoImpl runtime; + private void setNoMachinesForWorkspace(String workspaceId) throws EnvironmentNotRunningException { + when(envEngine.getMachines(workspaceId)).thenThrow(new EnvironmentNotRunningException("")); + } - public TestMachineInstance(Machine machine) { - super(machine); - runtime = mock(MachineRuntimeInfoImpl.class); + private List allowEnvironmentStart(Workspace workspace, + String envName, + TestAction beforeReturn) throws Exception { + Environment environment = workspace.getConfig().getEnvironments().get(envName); + ArrayList machines = new ArrayList<>(environment.getMachines().size()); + for (Map.Entry entry : environment.getMachines().entrySet()) { + machines.add(newMachine(workspace.getId(), + envName, + entry.getKey(), + entry.getValue().getAgents().contains("org.eclipse.che.ws-agent"))); } + when(envEngine.start(eq(workspace.getId()), + eq(envName), + eq(workspace.getConfig().getEnvironments().get(envName)), + anyBoolean(), + any(), + any())).thenAnswer(invocation -> { + if (beforeReturn != null) { + beforeReturn.call(); + } + return machines; + }); + return machines; + } + + private List allowEnvironmentStart(Workspace workspace, String envName) throws Exception { + return allowEnvironmentStart(workspace, envName, null); + } - @Override - public MachineRuntimeInfoImpl getRuntime() { - return runtime; + private void rejectEnvironmentStart(Workspace workspace, String envName, Exception x) throws Exception { + when(envEngine.start(eq(workspace.getId()), + eq(envName), + eq(workspace.getConfig().getEnvironments().get(envName)), + anyBoolean(), + any(), + any())).thenThrow(x); + } + + private void rejectEnvironmentStop(Workspace workspace, Exception x) throws Exception { + doThrow(x).when(envEngine).stop(workspace.getId()); + } + + private List prepareMachines(String workspaceId, String envName) throws EnvironmentNotRunningException { + List machines = new ArrayList<>(3); + machines.add(newMachine(workspaceId, envName, "machine1", true)); + machines.add(newMachine(workspaceId, envName, "machine2", false)); + machines.add(newMachine(workspaceId, envName, "machine3", false)); + prepareMachines(workspaceId, machines); + return machines; + } + + private void prepareMachines(String workspaceId, List machines) throws EnvironmentNotRunningException { + when(envEngine.getMachines(workspaceId)).thenReturn(machines); + } + + private Instance newMachine(String workspaceId, String envName, String name, boolean isDev) { + MachineImpl machine = MachineImpl.builder() + .setConfig(MachineConfigImpl.builder() + .setDev(isDev) + .setName(name) + .setType("docker") + .setSource(new MachineSourceImpl("type")) + .setLimits(new MachineLimitsImpl(1024)) + .build()) + .setWorkspaceId(workspaceId) + .setEnvName(envName) + .setOwner("owner") + .setRuntime(new MachineRuntimeInfoImpl(Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap())) + .setStatus(MachineStatus.RUNNING) + .build(); + return new NoOpMachineInstance(machine); + } + + private WorkspaceImpl newWorkspace(String workspaceId, String envName) { + EnvironmentImpl environment = new EnvironmentImpl(); + Map machines = environment.getMachines(); + machines.put("dev", new ExtendedMachineImpl(Arrays.asList("org.eclipse.che.terminal", + "org.eclipse.che.ws-agent"), + Collections.emptyMap(), + Collections.emptyMap())); + machines.put("db", new ExtendedMachineImpl(singletonList("org.eclipse.che.terminal"), + Collections.emptyMap(), + Collections.emptyMap())); + return WorkspaceImpl.builder() + .setId(workspaceId) + .setTemporary(false) + .setConfig(WorkspaceConfigImpl.builder() + .setName("test-workspace") + .setDescription("this is test workspace") + .setDefaultEnv(envName) + .setEnvironments(ImmutableMap.of(envName, + environment)) + .build()) + .build(); + } + + private void shutdownAndWaitPool(ExecutorService pool) throws InterruptedException { + pool.shutdownNow(); + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + fail("Can't shutdown test pool"); } } + + @FunctionalInterface + private interface TestAction { + void call() throws Exception; + } + + @DataProvider + private static Object[][] allStatuses() { + WorkspaceStatus[] values = WorkspaceStatus.values(); + WorkspaceStatus[][] result = new WorkspaceStatus[values.length][1]; + for (int i = 0; i < values.length; i++) { + result[i][0] = values[i]; + } + return result; + } + + @DataProvider + private static Object[][] notAllowedToStopStatuses() { + return new WorkspaceStatus[][] { + {WorkspaceStatus.STOPPING}, + {WorkspaceStatus.SNAPSHOTTING} + }; + } } diff --git a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceServiceTest.java b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceServiceTest.java index b0332b338d9..84f27df150a 100644 --- a/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceServiceTest.java +++ b/wsmaster/che-core-api-workspace/src/test/java/org/eclipse/che/api/workspace/server/WorkspaceServiceTest.java @@ -798,7 +798,7 @@ public void testWorkspaceLinks() throws Exception { .get(workspace.getConfig().getDefaultEnv()); assertNotNull(environment); - final WorkspaceRuntimeImpl runtime = new WorkspaceRuntimeImpl(workspace.getConfig().getDefaultEnv()); + final WorkspaceRuntimeImpl runtime = new WorkspaceRuntimeImpl(workspace.getConfig().getDefaultEnv(), null); MachineConfigImpl devMachineConfig = MachineConfigImpl.builder() .setDev(true) .setEnvVariables(emptyMap()) diff --git a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java index 0f4252640fb..f6ac7865b69 100644 --- a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java +++ b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java @@ -42,6 +42,7 @@ import org.eclipse.che.api.user.server.spi.UserDao; import org.eclipse.che.api.workspace.server.WorkspaceManager; import org.eclipse.che.api.workspace.server.WorkspaceRuntimes; +import org.eclipse.che.api.workspace.server.WorkspaceSharedPool; import org.eclipse.che.api.workspace.server.event.BeforeWorkspaceRemovedEvent; import org.eclipse.che.api.workspace.server.jpa.JpaWorkspaceDao.RemoveSnapshotsBeforeWorkspaceRemovedEventSubscriber; import org.eclipse.che.api.workspace.server.jpa.JpaWorkspaceDao.RemoveWorkspaceBeforeAccountRemovedEventSubscriber; @@ -142,6 +143,7 @@ protected void configure() { bind(AccountManager.class); bind(Boolean.class).annotatedWith(Names.named("che.workspace.auto_snapshot")).toInstance(false); bind(Boolean.class).annotatedWith(Names.named("che.workspace.auto_restore")).toInstance(false); + bind(WorkspaceSharedPool.class).toInstance(new WorkspaceSharedPool("cached", null, null)); } });