From d89ba9afe6544b232cec6e2a5683400e54c91e29 Mon Sep 17 00:00:00 2001 From: Dilshat Aliev Date: Tue, 6 Feb 2018 15:23:00 +0600 Subject: [PATCH 1/3] #2226 moved disk checking functionality into environment manager module to avoid security restrictions --- .../impl/EnvironmentManagerImpl.java | 33 ++++++++++++++ .../impl/adapter/EnvironmentAdapter.java | 6 +++ .../tasks/ContainerDiskUsageCheckTask.java} | 44 ++++++------------- .../core/hubmanager/impl/HubManagerImpl.java | 5 --- .../impl/adapter/HubAdapterImpl.java | 16 +++++++ .../subutai/hub/share/common/HubAdapter.java | 3 ++ 6 files changed, 71 insertions(+), 36 deletions(-) rename management/server/core/{hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/requestor/ContainerDiskUsageChecker.java => environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java} (63%) diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java index e65a90fb2cf..7dea4b562b2 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java @@ -102,6 +102,7 @@ import io.subutai.core.environment.impl.entity.EnvironmentAlertHandlerImpl; import io.subutai.core.environment.impl.entity.EnvironmentContainerImpl; import io.subutai.core.environment.impl.entity.LocalEnvironment; +import io.subutai.core.environment.impl.tasks.ContainerDiskUsageCheckTask; import io.subutai.core.environment.impl.tasks.EnvironmentManagerInitTask; import io.subutai.core.environment.impl.tasks.RemoveEnvironmentsTask; import io.subutai.core.environment.impl.tasks.UploadEnvironmentsTask; @@ -154,6 +155,7 @@ public class EnvironmentManagerImpl private static final long SYNC_ENVS_WITH_HUB_INTERVAL_MIN = 10; private static final String REMOTE_OWNER_NAME = "remote"; private static final String UKNOWN_OWNER_NAME = "unknown"; + private static final long CONTAINER_DISK_USAGE_CHECK_INTERVAL_MIN = 6 * 60; // 6 hrs private final IdentityManager identityManager; private final RelationManager relationManager; @@ -175,6 +177,7 @@ public class EnvironmentManagerImpl protected PGPKeyUtil pgpKeyUtil = new PGPKeyUtil(); private volatile long lastP2pSecretKeyResetTs = 0L; private volatile long lastEnvSyncTs = 0L; + private volatile long lastContainerDiskUsageCheckTs = 0L; public EnvironmentManagerImpl( final TemplateManager templateManager, final PeerManager peerManager, @@ -2340,11 +2343,33 @@ public void run() resetP2pKeys(); + checkContainerDiskUsage(); + LOG.debug( "Environment background tasks finished." ); } } + private void checkContainerDiskUsage() + { + if ( System.currentTimeMillis() - lastContainerDiskUsageCheckTs >= TimeUnit.MINUTES + .toMillis( CONTAINER_DISK_USAGE_CHECK_INTERVAL_MIN ) ) + { + lastContainerDiskUsageCheckTs = System.currentTimeMillis(); + + Subject.doAs( systemUser, new PrivilegedAction() + { + @Override + public Void run() + { + doCheckContainerDiskUsage(); + return null; + } + } ); + } + } + + private void resetP2pKeys() { if ( System.currentTimeMillis() - lastP2pSecretKeyResetTs >= TimeUnit.MINUTES @@ -2412,6 +2437,14 @@ public Void run() } + private void doCheckContainerDiskUsage() + { + getCachedExecutor().execute( + new ContainerDiskUsageCheckTask( environmentAdapter.getHubAdapter(), peerManager.getLocalPeer(), + this ) ); + } + + void uploadPeerOwnerEnvironmentsToHub() { getCachedExecutor() diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/adapter/EnvironmentAdapter.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/adapter/EnvironmentAdapter.java index 56e748527a0..aa295dacc6a 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/adapter/EnvironmentAdapter.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/adapter/EnvironmentAdapter.java @@ -62,6 +62,12 @@ public EnvironmentAdapter( EnvironmentManagerImpl environmentManager, PeerManage } + public HubAdapter getHubAdapter() + { + return hubAdapter; + } + + public HubEnvironment get( String id ) { try diff --git a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/requestor/ContainerDiskUsageChecker.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java similarity index 63% rename from management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/requestor/ContainerDiskUsageChecker.java rename to management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java index 6d0b0e8fb21..a8ff7c22707 100644 --- a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/requestor/ContainerDiskUsageChecker.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java @@ -1,4 +1,4 @@ -package io.subutai.core.hubmanager.impl.requestor; +package io.subutai.core.environment.impl.tasks; import java.util.concurrent.TimeUnit; @@ -15,34 +15,30 @@ import io.subutai.common.peer.LocalPeer; import io.subutai.common.settings.Common; import io.subutai.common.util.TaskUtil; -import io.subutai.core.environment.api.EnvironmentManager; -import io.subutai.core.hubmanager.api.HubManager; -import io.subutai.core.hubmanager.api.HubRequester; -import io.subutai.core.hubmanager.api.RestClient; -import io.subutai.core.hubmanager.api.RestResult; +import io.subutai.core.environment.impl.EnvironmentManagerImpl; +import io.subutai.hub.share.common.HubAdapter; -//https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota -public class ContainerDiskUsageChecker extends HubRequester +public class ContainerDiskUsageCheckTask implements Runnable { - private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageChecker.class ); - - private final EnvironmentManager environmentManager; + private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageCheckTask.class ); + private final EnvironmentManagerImpl environmentManager; + private final HubAdapter hubAdapter; private final LocalPeer localPeer; private final CommandUtil commandUtil = new CommandUtil(); - public ContainerDiskUsageChecker( final HubManager hubManager, final RestClient restClient, - final EnvironmentManager environmentManager, final LocalPeer localPeer ) + public ContainerDiskUsageCheckTask( final HubAdapter hubAdapter, final LocalPeer localPeer, + final EnvironmentManagerImpl environmentManager ) { - super( hubManager, restClient ); this.environmentManager = environmentManager; + this.hubAdapter = hubAdapter; this.localPeer = localPeer; } @Override - public void request() throws Exception + public void run() { for ( EnvironmentDto environment : environmentManager.getTenantEnvironments() ) { @@ -90,8 +86,8 @@ private void checkDiskUsage( ContainerDto containerDto ) } //notify Hub - notifyHub( containerDto.getPeerId(), containerDto.getEnvironmentId(), containerDto.getId(), diskUsed, - stop ); + hubAdapter.notifyContainerDiskUsageExcess( containerDto.getPeerId(), containerDto.getEnvironmentId(), + containerDto.getId(), diskUsed, stop ); } } catch ( Exception e ) @@ -99,18 +95,4 @@ private void checkDiskUsage( ContainerDto containerDto ) LOG.error( "Error checking disk usage of container " + containerDto.getContainerName(), e.getMessage() ); } } - - - private void notifyHub( String peerId, String envId, String contId, long diskUsage, boolean containerWasStopped ) - { - RestResult result = restClient.post( String - .format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId, - diskUsage, containerWasStopped ), null ); - - if ( !result.isSuccess() ) - { - LOG.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(), - result.getReasonPhrase() ); - } - } } diff --git a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/HubManagerImpl.java b/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/HubManagerImpl.java index e0b99c8cd38..757d6961235 100644 --- a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/HubManagerImpl.java +++ b/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/HubManagerImpl.java @@ -63,7 +63,6 @@ import io.subutai.core.hubmanager.impl.processor.ProxyProcessor; import io.subutai.core.hubmanager.impl.processor.UserTokenProcessor; import io.subutai.core.hubmanager.impl.processor.port_map.ContainerPortMapProcessor; -import io.subutai.core.hubmanager.impl.requestor.ContainerDiskUsageChecker; import io.subutai.core.hubmanager.impl.requestor.ContainerEventProcessor; import io.subutai.core.hubmanager.impl.requestor.ContainerMetricsProcessor; import io.subutai.core.hubmanager.impl.requestor.HubLoggerProcessor; @@ -236,10 +235,6 @@ private void initHubRequesters() requestorsRunner.scheduleWithFixedDelay( new ContainerMetricsProcessor( this, localPeer, monitor, restClient, containerMetricsService, CONTAINER_METRIC_SEND_INTERVAL_MIN ), 1, CONTAINER_METRIC_SEND_INTERVAL_MIN, TimeUnit.MINUTES ); - //*********** - requestorsRunner - .scheduleWithFixedDelay( new ContainerDiskUsageChecker( this, restClient, envManager, localPeer ), 10, - 6 * 60 /* 6 hours */, TimeUnit.MINUTES ); } diff --git a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/adapter/HubAdapterImpl.java b/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/adapter/HubAdapterImpl.java index efef6e0f881..8e6168ea1c1 100644 --- a/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/adapter/HubAdapterImpl.java +++ b/management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/adapter/HubAdapterImpl.java @@ -361,6 +361,22 @@ public boolean deletePluginData( String pluginKey, String key ) } + @Override + public void notifyContainerDiskUsageExcess( String peerId, String envId, String contId, long diskUsage, + boolean containerWasStopped ) + { + RestResult result = getRestClient().post( String + .format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId, + diskUsage, containerWasStopped ), null ); + + if ( !result.isSuccess() ) + { + log.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(), + result.getReasonPhrase() ); + } + } + + private void onContainerStateChange( String envId, String contId, String state ) { if ( !isRegistered() ) diff --git a/management/server/subutai-hub-share/src/main/java/io/subutai/hub/share/common/HubAdapter.java b/management/server/subutai-hub-share/src/main/java/io/subutai/hub/share/common/HubAdapter.java index 0509c0da2c4..000d4f8e948 100644 --- a/management/server/subutai-hub-share/src/main/java/io/subutai/hub/share/common/HubAdapter.java +++ b/management/server/subutai-hub-share/src/main/java/io/subutai/hub/share/common/HubAdapter.java @@ -39,4 +39,7 @@ public interface HubAdapter boolean uploadPluginData( String pluginKey, String key, Object data ); boolean deletePluginData( String pluginKey, String key ); + + void notifyContainerDiskUsageExcess( String peerId, String envId, String contId, long diskUsage, + boolean containerWasStopped ); } From c58c9de6b8d36f531cc775ac6ce4f36b4d9f9c7c Mon Sep 17 00:00:00 2001 From: Dilshat Aliev Date: Tue, 6 Feb 2018 19:05:34 +0600 Subject: [PATCH 2/3] #2226 fixed execution host --- .../impl/tasks/ContainerDiskUsageCheckTask.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java index a8ff7c22707..72eec6f5f03 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java @@ -13,6 +13,7 @@ import io.subutai.common.environment.EnvironmentDto; import io.subutai.common.peer.ContainerHost; import io.subutai.common.peer.LocalPeer; +import io.subutai.common.peer.ResourceHost; import io.subutai.common.settings.Common; import io.subutai.common.util.TaskUtil; import io.subutai.core.environment.impl.EnvironmentManagerImpl; @@ -65,13 +66,15 @@ private void checkDiskUsage( ContainerDto containerDto ) // b.b if du is >= 150 % of quota -> stop container, notify Hub try { + ResourceHost resourceHost = localPeer.getResourceHostById( containerDto.getRhId() ); + ContainerHost containerHost = localPeer.getContainerHostById( containerDto.getId() ); CommandResult result = commandUtil .execute( new RequestBuilder( "subutai info du " + containerDto.getContainerName() ), - containerHost ); + resourceHost ); - long diskUsed = Long.parseLong( result.getStdOut() ); + long diskUsed = Long.parseLong( result.getStdOut().trim() ); long diskLimit = containerHost.getContainerSize().getDiskQuota().longValue(); @@ -92,7 +95,8 @@ private void checkDiskUsage( ContainerDto containerDto ) } catch ( Exception e ) { - LOG.error( "Error checking disk usage of container " + containerDto.getContainerName(), e.getMessage() ); + LOG.error( "Error checking disk usage of container {}: {}", containerDto.getContainerName(), + e.getMessage() ); } } } From 6bd38b4289ee001195ed0abe9a73c435d8e8b89f Mon Sep 17 00:00:00 2001 From: Dilshat Aliev Date: Wed, 7 Feb 2018 08:58:05 +0600 Subject: [PATCH 3/3] #2226 skip stopped containers, added logs --- .../impl/tasks/ContainerDiskUsageCheckTask.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java index 72eec6f5f03..bf47c2c3ce7 100644 --- a/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java +++ b/management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java @@ -11,6 +11,7 @@ import io.subutai.common.command.RequestBuilder; import io.subutai.common.environment.ContainerDto; import io.subutai.common.environment.EnvironmentDto; +import io.subutai.common.host.ContainerHostState; import io.subutai.common.peer.ContainerHost; import io.subutai.common.peer.LocalPeer; import io.subutai.common.peer.ResourceHost; @@ -66,6 +67,12 @@ private void checkDiskUsage( ContainerDto containerDto ) // b.b if du is >= 150 % of quota -> stop container, notify Hub try { + + if ( containerDto.getState() != ContainerHostState.RUNNING ) + { + return; + } + ResourceHost resourceHost = localPeer.getResourceHostById( containerDto.getRhId() ); ContainerHost containerHost = localPeer.getContainerHostById( containerDto.getId() ); @@ -80,12 +87,17 @@ private void checkDiskUsage( ContainerDto containerDto ) if ( diskUsed >= diskLimit * 0.9 ) { + LOG.info( "Container {} is exceeding disk quota: limit {}, actual usage {}", + containerDto.getContainerName(), diskLimit, diskUsed ); + boolean stop = diskUsed >= diskLimit * 1.5; if ( stop ) { //stop container containerHost.stop(); + + LOG.info( "Container {} is stopped due to disk quota excess", containerDto.getContainerName() ); } //notify Hub