dedicated vc rest api

docs/rest-server/API.md

@@ -781,9 +781,9 @@ GET /api/v1/virtual-clusters/:vcName
 Status: 200
 
 {
-  //capacity percentage this virtual cluster can use of entire cluster
+  // capacity percentage this virtual cluster can use of entire cluster
   "capacity":50,
-  //max capacity percentage this virtual cluster can use of entire cluster
+  // max capacity percentage this virtual cluster can use of entire cluster
   "maxCapacity":100,
   // used capacity percentage this virtual cluster can use of entire cluster
   "usedCapacity":0,
@@ -795,7 +795,18 @@ Status: 200
    "vCores":0,
    "GPUs":0
   },
-  "state":"running"
+  "resourcesTotal":{
+   "memory":0,
+   "vCores":0,
+   "GPUs":0
+  },
+  "dedicated": true/false,
+  // available node list for this virtual cluster
+  "nodeList": [node1, node2, ...],
+  // RUNNING: vc is enabled
+  // STOPPED: vc is disabled, without either new job or running job.
+  // DRAINING: intermedia state from RUNNING to STOPPED, in waiting on existing job.
+  "status":"RUNNING"/"STOPPED"/"DRAINING"
 }
 ```
 

src/dev-box/build/dev-box.dockerfile

@@ -52,7 +52,7 @@ RUN apt-get -y update && \
       net-tools && \
     mkdir -p /cluster-configuration &&\
     git clone https://github.com/Microsoft/pai.git &&\
-    pip install python-etcd docker kubernetes GitPython jsonschema
+    pip install python-etcd docker kubernetes GitPython jsonschema attrs dicttoxml beautifulsoup4
 
 WORKDIR /tmp
 

src/hadoop-node-manager/deploy/hadoop-node-manager-configuration/hdfs-site.xml

@@ -20,7 +20,7 @@
 
 <property>
   <name>dfs.replication</name>
-  <value>2</value>
+  <value>3</value>
   <description>
          Default block replication. The actual number of replicas can be specified when
          the file is created. The default is used if replication is not specified at
@@ -234,7 +234,7 @@
           rejected. It is recommended that this setting be left on to prevent accidental
           registration of datanodes listed by hostname in the excludes file during a DNS
           outage. Only set this to false in environments where there is no infrastructure
-          to support reverse DNS lookup.  
+          to support reverse DNS lookup.
   </description>
 </property>
 

src/hadoop-resource-manager/deploy/hadoop-resource-manager-configuration/resourcemanager-generate-script.sh

@@ -23,15 +23,16 @@ cp  /hadoop-configuration/yarn-site.xml $HADOOP_CONF_DIR/yarn-site.xml
 cp  /hadoop-configuration/hadoop-env.sh $HADOOP_CONF_DIR/hadoop-env.sh
 cp  /hadoop-configuration/yarn-env.sh $HADOOP_CONF_DIR/yarn-env.sh
 cp  /hadoop-configuration/capacity-scheduler.xml $HADOOP_CONF_DIR/capacity-scheduler.xml
+cp  /hadoop-configuration/hdfs-site.xml $HADOOP_CONF_DIR/hdfs-site.xml
 
 
-sed  -i "s/{RESOURCEMANAGER_ADDRESS}/${RESOURCEMANAGER_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml 
+sed  -i "s/{RESOURCEMANAGER_ADDRESS}/${RESOURCEMANAGER_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml
 sed  -i "s/{ZOOKEEPER_QUORUM}/${ZOOKEEPER_QUORUM}/g" $HADOOP_CONF_DIR/yarn-site.xml
-sed  -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml 
+sed  -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml
 sed  -i "s/{LOGSERVER_ADDRESS}/${LOGSERVER_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml
 sed  -i "s/{TIMELINE_SERVER_ADDRESS}/${TIMELINE_SERVER_ADDRESS}/g" $HADOOP_CONF_DIR/yarn-site.xml
 
-sed  -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml 
+sed  -i "s/{HDFS_ADDRESS}/${HDFS_ADDRESS}/g" $HADOOP_CONF_DIR/core-site.xml
 
-sed  -i "s/{LOGSERVER_ADDRESS}/${LOGSERVER_ADDRESS}/g" $HADOOP_CONF_DIR/mapred-site.xml 
+sed  -i "s/{LOGSERVER_ADDRESS}/${LOGSERVER_ADDRESS}/g" $HADOOP_CONF_DIR/mapred-site.xml
 

src/rest-server/src/config/vc.js

@@ -21,12 +21,10 @@ const Joi = require('joi');
 // define the input schema for the 'update vc' api
 const vcPutInputSchema = Joi.object().keys({
   vcCapacity: Joi.number()
-    .integer()
     .min(0)
     .max(100)
     .required(),
   vcMaxCapacity: Joi.number()
-    .integer()
     .min(Joi.ref('vcCapacity'))
     .max(100)
     .optional(),

src/rest-server/src/models/vc.js

@@ -31,7 +31,7 @@ class VirtualCluster {
     function traverse(queueInfo, queueDict) {
       if (queueInfo.type === 'capacitySchedulerLeafQueueInfo') {
         let queueDefaultLabel = queueInfo.defaultNodeLabelExpression;
-        if (typeof(queueDefaultLabel) === 'undefined' || queueDefaultLabel === '<DEFAULT_PARTITION>') {
+        if (typeof queueDefaultLabel === 'undefined' || queueDefaultLabel === '<DEFAULT_PARTITION>') {
           queueDefaultLabel = '';
         }
         let defaultPartitionInfo = null;
@@ -50,8 +50,8 @@ class VirtualCluster {
         }
 
         queueDict[queueInfo.queueName] = {
-          capacity: Math.round(defaultPartitionInfo.absoluteCapacity),
-          maxCapacity: Math.round(defaultPartitionInfo.absoluteMaxCapacity),
+          capacity: defaultPartitionInfo.absoluteCapacity,
+          maxCapacity: defaultPartitionInfo.absoluteMaxCapacity,
           usedCapacity: defaultPartitionInfo.absoluteUsedCapacity,
           numActiveJobs: queueInfo.numActiveApplications,
           numJobs: queueInfo.numApplications,
@@ -91,6 +91,20 @@ class VirtualCluster {
     return resourceByLabel;
   }
 
+  getNodesByLabel(nodeInfo) {
+    let nodesByLabel = {};
+    for (let node of nodeInfo) {
+      let nodeLabel = node.nodeLabels || [''];
+      nodeLabel = nodeLabel[0];
+      let nodeHostName = node.nodeHostName;
+      if (!nodesByLabel.hasOwnProperty(nodeLabel)) {
+        nodesByLabel[nodeLabel] = [];
+      }
+      nodesByLabel[nodeLabel].push(nodeHostName);
+    }
+    return nodesByLabel;
+  }
+
   addDedicatedInfo(vcInfo, next) {
     unirest.get(yarnConfig.yarnNodeInfoPath)
       .headers(yarnConfig.webserviceRequestHeaders)
@@ -100,20 +114,23 @@ class VirtualCluster {
             res.body : JSON.parse(res.body);
           const nodeInfo = resJson.nodes.node;
           let labeledResource = this.getResourceByLabel(nodeInfo);
+          let labeledNodes = this.getNodesByLabel(nodeInfo);
           for (let vcName of Object.keys(vcInfo)) {
             let resourcesTotal = {
               vCores: 0,
               memory: 0,
               GPUs: 0,
             };
             let vcLabel = vcInfo[vcName].defaultLabel;
+            delete vcInfo[vcName].defaultLabel;
             if (labeledResource.hasOwnProperty(vcLabel)) {
               let p = vcInfo[vcName].capacity;
               resourcesTotal.vCores = labeledResource[vcLabel].vCores * p / 100;
               resourcesTotal.memory = labeledResource[vcLabel].memory * p / 100;
               resourcesTotal.GPUs = labeledResource[vcLabel].GPUs * p / 100;
             }
             vcInfo[vcName].resourcesTotal = resourcesTotal;
+            vcInfo[vcName].nodeList = labeledNodes[vcLabel] || [];
           }
           next(vcInfo, null);
         } catch (error) {
@@ -214,13 +231,22 @@ class VirtualCluster {
         if (!vcList.hasOwnProperty('default')) {
           return callback(createError('Not Found', 'NoVirtualClusterError', `No default vc found, can't allocate quota`));
         } else {
-          let defaultQuotaIfUpdated = vcList['default']['capacity'] + (vcList[vcName] ? vcList[vcName]['capacity'] : 0) - capacity;
+          // let defaultQuotaIfUpdated = vcList['default']['capacity'] + (vcList[vcName] ? vcList[vcName]['capacity'] : 0) - capacity;
+          let defaultQuotaIfUpdated = 100.0;
+          defaultQuotaIfUpdated -= capacity;
+          for (let vc of Object.keys(vcList)) {
+            if (vc !== vcName && vc !== 'default' && vcList[vc].dedicated === false) {
+              defaultQuotaIfUpdated -= vcList[vc].capacity;
+            }
+          }
           if (defaultQuotaIfUpdated < 0) {
             return callback(createError('Forbidden', 'NoEnoughQuotaError', `No enough quota`));
           }
-
           let data = {'add-queue': {}, 'update-queue': {}};
           if (vcList.hasOwnProperty(vcName)) {
+            if (vcList[vcName].dedicated) {
+              return callback(createError('Forbidden', 'ReadOnlyVcError', `Dedicated vc is read-only, can't be updated by rest-api`));
+            }
             data['update-queue'][vcName] = {
               'capacity': capacity,
               'maximum-capacity': maxCapacity,
@@ -235,6 +261,9 @@ class VirtualCluster {
             'capacity': defaultQuotaIfUpdated,
             'maximum-capacity': defaultQuotaIfUpdated,
           };
+          if (vcList.default.maxCapacity === 100 || vcList.default.maxCapacity > vcList.default.capacity) {
+            data['update-queue']['default']['maximum-capacity'] = 100;
+          }
 
           // logger.debug('raw data to generate: ', data);
           const vcdataXml = this.generateUpdateInfo(data);
@@ -331,6 +360,8 @@ class VirtualCluster {
           return callback(createError('Not Found', 'NoVirtualClusterError', `No default vc found, can't free quota`));
         } else if (!vcList.hasOwnProperty(vcName)) {
           return callback(createError('Not Found', 'NoVirtualClusterError', `Can't delete a nonexistent vc ${vcName}`));
+        } else if (vcList[vcName].dedicated) {
+          return callback(createError('Forbidden', 'ReadOnlyVcError', `Dedicated vc is read-only, can't be removed by rest-api`));
         } else if (vcList[vcName]['numJobs'] > 0) {
           return callback(createError('Forbidden', 'RemoveRunningVcError',
             `Can't delete vc ${vcName}, ${vcList[vcName]['numJobs']} jobs are running, stop them before delete vc`));
@@ -339,7 +370,12 @@ class VirtualCluster {
             if (err) {
               return callback(err);
             } else {
-              let defaultQuotaIfUpdated = vcList['default']['capacity'] + vcList[vcName]['capacity'];
+              let defaultQuotaIfUpdated = 100.0;
+              for (let vc of Object.keys(vcList)) {
+                if (vc !== vcName && vc !== 'default' && vcList[vc].dedicated === false) {
+                  defaultQuotaIfUpdated -= vcList[vc].capacity;
+                }
+              }
               let data = {
                 'update-queue': {
                   [vcName]: {
@@ -355,6 +391,9 @@ class VirtualCluster {
                   [vcName]: null,
                 },
               };
+              if (vcList.default.maxCapacity === 100 || vcList.default.maxCapacity > vcList.default.capacity) {
+                data['update-queue']['default']['maximum-capacity'] = 100;
+              }
 
               // logger.debug('Raw data to generate: ', data);
               const vcdataXml = this.generateUpdateInfo(data);

src/rest-server/src/util/error.d.ts

@@ -41,6 +41,7 @@ declare type Code =
     'NoVirtualClusterError' |
     'ReadOnlyJobError' |
     'RemoveAdminError' |
+    'ReadOnlyVcError' |
     'RemoveRunningVcError' |
     'UnauthorizedUserError' |
     'NoEnoughQuotaError' |