Dev

management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/EnvironmentManagerImpl.java

@@ -102,6 +102,7 @@
 import io.subutai.core.environment.impl.entity.EnvironmentAlertHandlerImpl;
 import io.subutai.core.environment.impl.entity.EnvironmentContainerImpl;
 import io.subutai.core.environment.impl.entity.LocalEnvironment;
+import io.subutai.core.environment.impl.tasks.ContainerDiskUsageCheckTask;
 import io.subutai.core.environment.impl.tasks.EnvironmentManagerInitTask;
 import io.subutai.core.environment.impl.tasks.RemoveEnvironmentsTask;
 import io.subutai.core.environment.impl.tasks.UploadEnvironmentsTask;
@@ -154,6 +155,7 @@ public class EnvironmentManagerImpl
     private static final long SYNC_ENVS_WITH_HUB_INTERVAL_MIN = 10;
     private static final String REMOTE_OWNER_NAME = "remote";
     private static final String UKNOWN_OWNER_NAME = "unknown";
+    private static final long CONTAINER_DISK_USAGE_CHECK_INTERVAL_MIN = 6 * 60; // 6 hrs
 
     private final IdentityManager identityManager;
     private final RelationManager relationManager;
@@ -175,6 +177,7 @@ public class EnvironmentManagerImpl
     protected PGPKeyUtil pgpKeyUtil = new PGPKeyUtil();
     private volatile long lastP2pSecretKeyResetTs = 0L;
     private volatile long lastEnvSyncTs = 0L;
+    private volatile long lastContainerDiskUsageCheckTs = 0L;
 
 
     public EnvironmentManagerImpl( final TemplateManager templateManager, final PeerManager peerManager,
@@ -2340,11 +2343,33 @@ public void run()
 
             resetP2pKeys();
 
+            checkContainerDiskUsage();
+
             LOG.debug( "Environment background tasks finished." );
         }
     }
 
 
+    private void checkContainerDiskUsage()
+    {
+        if ( System.currentTimeMillis() - lastContainerDiskUsageCheckTs >= TimeUnit.MINUTES
+                .toMillis( CONTAINER_DISK_USAGE_CHECK_INTERVAL_MIN ) )
+        {
+            lastContainerDiskUsageCheckTs = System.currentTimeMillis();
+
+            Subject.doAs( systemUser, new PrivilegedAction<Void>()
+            {
+                @Override
+                public Void run()
+                {
+                    doCheckContainerDiskUsage();
+                    return null;
+                }
+            } );
+        }
+    }
+
+
     private void resetP2pKeys()
     {
         if ( System.currentTimeMillis() - lastP2pSecretKeyResetTs >= TimeUnit.MINUTES
@@ -2412,6 +2437,14 @@ public Void run()
     }
 
 
+    private void doCheckContainerDiskUsage()
+    {
+        getCachedExecutor().execute(
+                new ContainerDiskUsageCheckTask( environmentAdapter.getHubAdapter(), peerManager.getLocalPeer(),
+                        this ) );
+    }
+
+
     void uploadPeerOwnerEnvironmentsToHub()
     {
         getCachedExecutor()

management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/adapter/EnvironmentAdapter.java

@@ -62,6 +62,12 @@ public EnvironmentAdapter( EnvironmentManagerImpl environmentManager, PeerManage
     }
 
 
+    public HubAdapter getHubAdapter()
+    {
+        return hubAdapter;
+    }
+
+
     public HubEnvironment get( String id )
     {
         try

management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/requestor/ContainerDiskUsageChecker.java → management/server/core/environment-manager/environment-manager-impl/src/main/java/io/subutai/core/environment/impl/tasks/ContainerDiskUsageCheckTask.java

@@ -1,4 +1,4 @@
-package io.subutai.core.hubmanager.impl.requestor;
+package io.subutai.core.environment.impl.tasks;
 
 
 import java.util.concurrent.TimeUnit;
@@ -11,38 +11,36 @@
 import io.subutai.common.command.RequestBuilder;
 import io.subutai.common.environment.ContainerDto;
 import io.subutai.common.environment.EnvironmentDto;
+import io.subutai.common.host.ContainerHostState;
 import io.subutai.common.peer.ContainerHost;
 import io.subutai.common.peer.LocalPeer;
+import io.subutai.common.peer.ResourceHost;
 import io.subutai.common.settings.Common;
 import io.subutai.common.util.TaskUtil;
-import io.subutai.core.environment.api.EnvironmentManager;
-import io.subutai.core.hubmanager.api.HubManager;
-import io.subutai.core.hubmanager.api.HubRequester;
-import io.subutai.core.hubmanager.api.RestClient;
-import io.subutai.core.hubmanager.api.RestResult;
+import io.subutai.core.environment.impl.EnvironmentManagerImpl;
+import io.subutai.hub.share.common.HubAdapter;
 
 
-//https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota
-public class ContainerDiskUsageChecker extends HubRequester
+public class ContainerDiskUsageCheckTask implements Runnable
 {
-    private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageChecker.class );
-
-    private final EnvironmentManager environmentManager;
+    private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageCheckTask.class );
+    private final EnvironmentManagerImpl environmentManager;
+    private final HubAdapter hubAdapter;
     private final LocalPeer localPeer;
     private final CommandUtil commandUtil = new CommandUtil();
 
 
-    public ContainerDiskUsageChecker( final HubManager hubManager, final RestClient restClient,
-                                      final EnvironmentManager environmentManager, final LocalPeer localPeer )
+    public ContainerDiskUsageCheckTask( final HubAdapter hubAdapter, final LocalPeer localPeer,
+                                        final EnvironmentManagerImpl environmentManager )
     {
-        super( hubManager, restClient );
         this.environmentManager = environmentManager;
+        this.hubAdapter = hubAdapter;
         this.localPeer = localPeer;
     }
 
 
     @Override
-    public void request() throws Exception
+    public void run()
     {
         for ( EnvironmentDto environment : environmentManager.getTenantEnvironments() )
         {
@@ -69,48 +67,48 @@ private void checkDiskUsage( ContainerDto containerDto )
         //  b.b if du is >= 150 %  of quota -> stop container, notify Hub
         try
         {
+
+            if ( containerDto.getState() != ContainerHostState.RUNNING )
+            {
+                return;
+            }
+
+            ResourceHost resourceHost = localPeer.getResourceHostById( containerDto.getRhId() );
+
             ContainerHost containerHost = localPeer.getContainerHostById( containerDto.getId() );
 
             CommandResult result = commandUtil
                     .execute( new RequestBuilder( "subutai info du " + containerDto.getContainerName() ),
-                            containerHost );
+                            resourceHost );
 
-            long diskUsed = Long.parseLong( result.getStdOut() );
+            long diskUsed = Long.parseLong( result.getStdOut().trim() );
 
             long diskLimit = containerHost.getContainerSize().getDiskQuota().longValue();
 
             if ( diskUsed >= diskLimit * 0.9 )
             {
+                LOG.info( "Container {} is exceeding disk quota: limit {}, actual usage {}",
+                        containerDto.getContainerName(), diskLimit, diskUsed );
+
                 boolean stop = diskUsed >= diskLimit * 1.5;
 
                 if ( stop )
                 {
                     //stop container
                     containerHost.stop();
+
+                    LOG.info( "Container {} is stopped due to disk quota excess", containerDto.getContainerName() );
                 }
 
                 //notify Hub
-                notifyHub( containerDto.getPeerId(), containerDto.getEnvironmentId(), containerDto.getId(), diskUsed,
-                        stop );
+                hubAdapter.notifyContainerDiskUsageExcess( containerDto.getPeerId(), containerDto.getEnvironmentId(),
+                        containerDto.getId(), diskUsed, stop );
             }
         }
         catch ( Exception e )
         {
-            LOG.error( "Error checking disk usage of container " + containerDto.getContainerName(), e.getMessage() );
-        }
-    }
-
-
-    private void notifyHub( String peerId, String envId, String contId, long diskUsage, boolean containerWasStopped )
-    {
-        RestResult result = restClient.post( String
-                .format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId,
-                        diskUsage, containerWasStopped ), null );
-
-        if ( !result.isSuccess() )
-        {
-            LOG.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(),
-                    result.getReasonPhrase() );
+            LOG.error( "Error checking disk usage of container {}: {}", containerDto.getContainerName(),
+                    e.getMessage() );
         }
     }
 }

management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/HubManagerImpl.java

@@ -63,7 +63,6 @@
 import io.subutai.core.hubmanager.impl.processor.ProxyProcessor;
 import io.subutai.core.hubmanager.impl.processor.UserTokenProcessor;
 import io.subutai.core.hubmanager.impl.processor.port_map.ContainerPortMapProcessor;
-import io.subutai.core.hubmanager.impl.requestor.ContainerDiskUsageChecker;
 import io.subutai.core.hubmanager.impl.requestor.ContainerEventProcessor;
 import io.subutai.core.hubmanager.impl.requestor.ContainerMetricsProcessor;
 import io.subutai.core.hubmanager.impl.requestor.HubLoggerProcessor;
@@ -236,10 +235,6 @@ private void initHubRequesters()
         requestorsRunner.scheduleWithFixedDelay(
                 new ContainerMetricsProcessor( this, localPeer, monitor, restClient, containerMetricsService,
                         CONTAINER_METRIC_SEND_INTERVAL_MIN ), 1, CONTAINER_METRIC_SEND_INTERVAL_MIN, TimeUnit.MINUTES );
-        //***********
-        requestorsRunner
-                .scheduleWithFixedDelay( new ContainerDiskUsageChecker( this, restClient, envManager, localPeer ), 10,
-                        6 * 60 /* 6 hours */, TimeUnit.MINUTES );
     }
 
 

management/server/core/hub-manager/hub-manager-impl/src/main/java/io/subutai/core/hubmanager/impl/adapter/HubAdapterImpl.java

@@ -361,6 +361,22 @@ public boolean deletePluginData( String pluginKey, String key )
     }
 
 
+    @Override
+    public void notifyContainerDiskUsageExcess( String peerId, String envId, String contId, long diskUsage,
+                                                boolean containerWasStopped )
+    {
+        RestResult result = getRestClient().post( String
+                .format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId,
+                        diskUsage, containerWasStopped ), null );
+
+        if ( !result.isSuccess() )
+        {
+            log.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(),
+                    result.getReasonPhrase() );
+        }
+    }
+
+
     private void onContainerStateChange( String envId, String contId, String state )
     {
         if ( !isRegistered() )

management/server/subutai-hub-share/src/main/java/io/subutai/hub/share/common/HubAdapter.java

@@ -39,4 +39,7 @@ public interface HubAdapter
     boolean uploadPluginData( String pluginKey, String key, Object data );
 
     boolean deletePluginData( String pluginKey, String key );
+
+    void notifyContainerDiskUsageExcess( String peerId, String envId, String contId, long diskUsage,
+                                         boolean containerWasStopped );
 }