Skip to content

Commit

Permalink
Merge pull request #2234 from subutai-io/base-2226
Browse files Browse the repository at this point in the history
Base 2226
  • Loading branch information
Dilshat authored Feb 5, 2018
2 parents 2c511bc + b77bdca commit 7f1edd9
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,8 @@ public ContainerToken verifyToken( final String token, String containerHostId, S

if ( containerToken == null )
{
LOG.error( "Failed to verify container token " + token + " for host " + containerHostId );

throw new HostRegistrationException( "Couldn't verify container token" );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import io.subutai.core.hubmanager.impl.processor.ProxyProcessor;
import io.subutai.core.hubmanager.impl.processor.UserTokenProcessor;
import io.subutai.core.hubmanager.impl.processor.port_map.ContainerPortMapProcessor;
import io.subutai.core.hubmanager.impl.requestor.ContainerDiskUsageChecker;
import io.subutai.core.hubmanager.impl.requestor.ContainerEventProcessor;
import io.subutai.core.hubmanager.impl.requestor.ContainerMetricsProcessor;
import io.subutai.core.hubmanager.impl.requestor.HubLoggerProcessor;
Expand Down Expand Up @@ -232,11 +233,13 @@ private void initHubRequesters()
requestorsRunner.scheduleWithFixedDelay( environmentTelemetryProcessor, 20, 1800, TimeUnit.SECONDS );

//***********
final ContainerMetricsProcessor containersMetricsProcessor =
requestorsRunner.scheduleWithFixedDelay(
new ContainerMetricsProcessor( this, localPeer, monitor, restClient, containerMetricsService,
CONTAINER_METRIC_SEND_INTERVAL_MIN );
requestorsRunner.scheduleWithFixedDelay( containersMetricsProcessor, 1, CONTAINER_METRIC_SEND_INTERVAL_MIN,
TimeUnit.MINUTES );
CONTAINER_METRIC_SEND_INTERVAL_MIN ), 1, CONTAINER_METRIC_SEND_INTERVAL_MIN, TimeUnit.MINUTES );
//***********
requestorsRunner
.scheduleWithFixedDelay( new ContainerDiskUsageChecker( this, restClient, envManager, localPeer ), 1,
6 * 60 /* 6 hours */, TimeUnit.MINUTES );
}


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package io.subutai.core.hubmanager.impl.requestor;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.subutai.common.command.CommandResult;
import io.subutai.common.command.CommandUtil;
import io.subutai.common.command.RequestBuilder;
import io.subutai.common.environment.ContainerDto;
import io.subutai.common.environment.EnvironmentDto;
import io.subutai.common.peer.ContainerHost;
import io.subutai.common.peer.LocalPeer;
import io.subutai.common.settings.Common;
import io.subutai.core.environment.api.EnvironmentManager;
import io.subutai.core.hubmanager.api.HubManager;
import io.subutai.core.hubmanager.api.HubRequester;
import io.subutai.core.hubmanager.api.RestClient;
import io.subutai.core.hubmanager.api.RestResult;


//https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota
public class ContainerDiskUsageChecker extends HubRequester
{
private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageChecker.class );

private final EnvironmentManager environmentManager;
private final LocalPeer localPeer;
private final CommandUtil commandUtil = new CommandUtil();


public ContainerDiskUsageChecker( final HubManager hubManager, final RestClient restClient,
final EnvironmentManager environmentManager, final LocalPeer localPeer )
{
super( hubManager, restClient );
this.environmentManager = environmentManager;
this.localPeer = localPeer;
}


@Override
public void request() throws Exception
{
for ( EnvironmentDto environment : environmentManager.getTenantEnvironments() )
{
//iterate over hub containers
if ( Common.HUB_ID.equals( environment.getDataSource() ) )
{
for ( ContainerDto container : environment.getContainers() )
{
checkDiskUsage( container );
}
}
}
}


private void checkDiskUsage( ContainerDto containerDto )
{
// a. get its disk usage -> "subutai info du foo"
// b. compare with container size disk quota limit
// b.a if du is >= 90 % of quota -> notify Hub
// b.b if du is >= 150 % of quota -> stop container, notify Hub
try
{
ContainerHost containerHost = localPeer.getContainerHostById( containerDto.getId() );

CommandResult result = commandUtil
.execute( new RequestBuilder( "subutai info du " + containerDto.getContainerName() ),
containerHost );

long diskUsed = Long.parseLong( result.getStdOut() );

long diskLimit = containerHost.getContainerSize().getDiskQuota().longValue();

if ( diskUsed >= diskLimit * 0.9 )
{
boolean stop = diskUsed >= diskLimit * 1.5;

if ( stop )
{
//stop container
containerHost.stop();
}

//notify Hub
notifyHub( containerDto.getPeerId(), containerDto.getEnvironmentId(), containerDto.getId(), diskUsed,
stop );
}
}
catch ( Exception e )
{
LOG.error( "Error checking disk usage of container " + containerDto.getContainerName(), e.getMessage() );
}
}


private void notifyHub( String peerId, String envId, String contId, long diskUsage, boolean containerWasStopped )
{
RestResult result = restClient.post( String
.format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId,
diskUsage, containerWasStopped ), null );

if ( !result.isSuccess() )
{
LOG.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(),
result.getReasonPhrase() );
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.subutai.common.task.CommandBatch;
import io.subutai.hub.share.quota.ContainerQuota;
import io.subutai.hub.share.quota.Quota;
import io.subutai.hub.share.resource.ByteUnit;
import io.subutai.hub.share.resource.ContainerResourceType;


Expand Down Expand Up @@ -54,15 +55,26 @@ public static RequestBuilder getSetQuotaCommand( String containerName, Container
quotaCommand.addArgument( containerName );
quotaCommand.addArgument( r.getResource().getContainerResourceType().getKey() );
quotaCommand.addArgument( "-s" );
quotaCommand.addArgument( r.getResource().getWriteValue() );

if ( r.getResource().getContainerResourceType() == ContainerResourceType.DISK )
{
//temp workaround for btrfs quota issue https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota

quotaCommand.addArgument( String.valueOf( r.getAsDiskResource().longValue( ByteUnit.GB ) * 2 ) );
}
else
{
quotaCommand.addArgument( r.getResource().getWriteValue() );
}

if ( r.getThreshold() != null && r.getThreshold() != 0 && !(
r.getResource().getContainerResourceType() == ContainerResourceType.CPUSET
//TODO fix atm threshold for disk is not supported and results in error on system level
|| r.getResource().getContainerResourceType() == ContainerResourceType.DISK ) )
{
quotaCommand.addArgument( "-t" );
quotaCommand.addArgument( r.getThreshold().toString() );
}

result.addCommand( quotaCommand );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
<div class="b-main-form__wrapper b-main-form__wrapper_min">
<label class="b-popup-body-label b-popup-body-label_inline">
<i class="fa fa-question-circle" style="font-size:16px"
tooltips tooltip-side="bottom" tooltip-template='Command execution before killed in seconds'></i>&nbsp;Timeout</label>
tooltips tooltip-side="bottom" tooltip-template='Command execution timeout in seconds'></i>&nbsp;Timeout</label>
<input type="text"
ng-model="consoleViewCtrl.timeOut"
class="b-popup-body-input">
Expand Down

0 comments on commit 7f1edd9

Please sign in to comment.