Skip to content

Commit

Permalink
Merge pull request #2235 from subutai-io/sysnet
Browse files Browse the repository at this point in the history
Sysnet
  • Loading branch information
Dilshat authored Feb 5, 2018
2 parents 62e77a2 + 7f1edd9 commit ddbf956
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,6 @@ public Response executeCommand( final String hostId, final String command, Strin
{
throw new AccessControlException( "Access denied" );
}
catch ( Exception e )
{
LOG.error( "Error executing command", e );

return Response.status( Response.Status.INTERNAL_SERVER_ERROR )
.entity( JsonUtil.toJson( e.getMessage() ) ).build();
}
}
// this command is intended to run on RH
else
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,8 @@ public ContainerToken verifyToken( final String token, String containerHostId, S

if ( containerToken == null )
{
LOG.error( "Failed to verify container token " + token + " for host " + containerHostId );

throw new HostRegistrationException( "Couldn't verify container token" );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import io.subutai.core.hubmanager.impl.processor.ProxyProcessor;
import io.subutai.core.hubmanager.impl.processor.UserTokenProcessor;
import io.subutai.core.hubmanager.impl.processor.port_map.ContainerPortMapProcessor;
import io.subutai.core.hubmanager.impl.requestor.ContainerDiskUsageChecker;
import io.subutai.core.hubmanager.impl.requestor.ContainerEventProcessor;
import io.subutai.core.hubmanager.impl.requestor.ContainerMetricsProcessor;
import io.subutai.core.hubmanager.impl.requestor.HubLoggerProcessor;
Expand Down Expand Up @@ -232,11 +233,13 @@ private void initHubRequesters()
requestorsRunner.scheduleWithFixedDelay( environmentTelemetryProcessor, 20, 1800, TimeUnit.SECONDS );

//***********
final ContainerMetricsProcessor containersMetricsProcessor =
requestorsRunner.scheduleWithFixedDelay(
new ContainerMetricsProcessor( this, localPeer, monitor, restClient, containerMetricsService,
CONTAINER_METRIC_SEND_INTERVAL_MIN );
requestorsRunner.scheduleWithFixedDelay( containersMetricsProcessor, 1, CONTAINER_METRIC_SEND_INTERVAL_MIN,
TimeUnit.MINUTES );
CONTAINER_METRIC_SEND_INTERVAL_MIN ), 1, CONTAINER_METRIC_SEND_INTERVAL_MIN, TimeUnit.MINUTES );
//***********
requestorsRunner
.scheduleWithFixedDelay( new ContainerDiskUsageChecker( this, restClient, envManager, localPeer ), 1,
6 * 60 /* 6 hours */, TimeUnit.MINUTES );
}


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package io.subutai.core.hubmanager.impl.requestor;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.subutai.common.command.CommandResult;
import io.subutai.common.command.CommandUtil;
import io.subutai.common.command.RequestBuilder;
import io.subutai.common.environment.ContainerDto;
import io.subutai.common.environment.EnvironmentDto;
import io.subutai.common.peer.ContainerHost;
import io.subutai.common.peer.LocalPeer;
import io.subutai.common.settings.Common;
import io.subutai.core.environment.api.EnvironmentManager;
import io.subutai.core.hubmanager.api.HubManager;
import io.subutai.core.hubmanager.api.HubRequester;
import io.subutai.core.hubmanager.api.RestClient;
import io.subutai.core.hubmanager.api.RestResult;


//https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota
public class ContainerDiskUsageChecker extends HubRequester
{
private final static Logger LOG = LoggerFactory.getLogger( ContainerDiskUsageChecker.class );

private final EnvironmentManager environmentManager;
private final LocalPeer localPeer;
private final CommandUtil commandUtil = new CommandUtil();


public ContainerDiskUsageChecker( final HubManager hubManager, final RestClient restClient,
final EnvironmentManager environmentManager, final LocalPeer localPeer )
{
super( hubManager, restClient );
this.environmentManager = environmentManager;
this.localPeer = localPeer;
}


@Override
public void request() throws Exception
{
for ( EnvironmentDto environment : environmentManager.getTenantEnvironments() )
{
//iterate over hub containers
if ( Common.HUB_ID.equals( environment.getDataSource() ) )
{
for ( ContainerDto container : environment.getContainers() )
{
checkDiskUsage( container );
}
}
}
}


private void checkDiskUsage( ContainerDto containerDto )
{
// a. get its disk usage -> "subutai info du foo"
// b. compare with container size disk quota limit
// b.a if du is >= 90 % of quota -> notify Hub
// b.b if du is >= 150 % of quota -> stop container, notify Hub
try
{
ContainerHost containerHost = localPeer.getContainerHostById( containerDto.getId() );

CommandResult result = commandUtil
.execute( new RequestBuilder( "subutai info du " + containerDto.getContainerName() ),
containerHost );

long diskUsed = Long.parseLong( result.getStdOut() );

long diskLimit = containerHost.getContainerSize().getDiskQuota().longValue();

if ( diskUsed >= diskLimit * 0.9 )
{
boolean stop = diskUsed >= diskLimit * 1.5;

if ( stop )
{
//stop container
containerHost.stop();
}

//notify Hub
notifyHub( containerDto.getPeerId(), containerDto.getEnvironmentId(), containerDto.getId(), diskUsed,
stop );
}
}
catch ( Exception e )
{
LOG.error( "Error checking disk usage of container " + containerDto.getContainerName(), e.getMessage() );
}
}


private void notifyHub( String peerId, String envId, String contId, long diskUsage, boolean containerWasStopped )
{
RestResult result = restClient.post( String
.format( "/rest/v1/peers/%s/environments/%s/containers/%s/disk_usage/%d/%s", peerId, envId, contId,
diskUsage, containerWasStopped ), null );

if ( !result.isSuccess() )
{
LOG.error( "Error notifying Hub about container disk usage excess: HTTP {} - {}", result.getStatus(),
result.getReasonPhrase() );
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.subutai.common.task.CommandBatch;
import io.subutai.hub.share.quota.ContainerQuota;
import io.subutai.hub.share.quota.Quota;
import io.subutai.hub.share.resource.ByteUnit;
import io.subutai.hub.share.resource.ContainerResourceType;


Expand Down Expand Up @@ -54,15 +55,26 @@ public static RequestBuilder getSetQuotaCommand( String containerName, Container
quotaCommand.addArgument( containerName );
quotaCommand.addArgument( r.getResource().getContainerResourceType().getKey() );
quotaCommand.addArgument( "-s" );
quotaCommand.addArgument( r.getResource().getWriteValue() );

if ( r.getResource().getContainerResourceType() == ContainerResourceType.DISK )
{
//temp workaround for btrfs quota issue https://github.com/subutai-io/agent/wiki/Switch-to-Soft-Quota

quotaCommand.addArgument( String.valueOf( r.getAsDiskResource().longValue( ByteUnit.GB ) * 2 ) );
}
else
{
quotaCommand.addArgument( r.getResource().getWriteValue() );
}

if ( r.getThreshold() != null && r.getThreshold() != 0 && !(
r.getResource().getContainerResourceType() == ContainerResourceType.CPUSET
//TODO fix atm threshold for disk is not supported and results in error on system level
|| r.getResource().getContainerResourceType() == ContainerResourceType.DISK ) )
{
quotaCommand.addArgument( "-t" );
quotaCommand.addArgument( r.getThreshold().toString() );
}

result.addCommand( quotaCommand );
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class Common

public static final String PEER_CERT_ALIAS = "peer_cert";

public static final int WAIT_CONTAINER_CONNECTION_SEC = 50;
public static final int WAIT_CONTAINER_CONNECTION_SEC = 60;
public static final int DEFAULT_EXECUTOR_REQUEST_TIMEOUT_SEC = 30;
public static final int MIN_COMMAND_TIMEOUT_SEC = 1;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@
</svg>
</span><!--
--><span class="b-nav-menu__title b-nav-menu-link__item">
Bazaar
Plugins
</span>
</a>
</li>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,23 +75,11 @@
</div>

<div class="b-toggle-buttons">
<!--<ul>
<li class="b-toggle-buttons__item"
ng-class="consoleViewCtrl.currentType == 'peer' ? 'b-toggle-buttons__item_active' : ''"
ng-click="consoleViewCtrl.setCurrentType('peer')">
Peer
</li>
<li class="b-toggle-buttons__item"
ng-class="consoleViewCtrl.currentType == 'environments' ? 'b-toggle-buttons__item_active' : ''"
ng-click="consoleViewCtrl.setCurrentType('environments')">
Environment
</li>
</ul>-->

<div class="b-main-form__wrapper b-main-form__wrapper_min">
<label class="b-popup-body-label b-popup-body-label_inline">
<i class="fa fa-question-circle" style="font-size:16px"
tooltips tooltip-side="bottom" tooltip-template='Command execution before killed in seconds'></i>&nbsp;Timeout</label>
tooltips tooltip-side="bottom" tooltip-template='Command execution timeout in seconds'></i>&nbsp;Timeout</label>
<input type="text"
ng-model="consoleViewCtrl.timeOut"
class="b-popup-body-input">
Expand Down

0 comments on commit ddbf956

Please sign in to comment.