Merge pull request #1242 from eiffel777/cloud-utilization-metrics

Add cloud core hour utilization statistic
ubccr · Jun 29, 2020 · 478505c · 478505c
2 parents c525399 + 53b49fd
commit 478505c
Show file tree

Hide file tree

Showing 177 changed files with 2,457 additions and 534 deletions.
diff --git a/bin/xdmod-ingestor b/bin/xdmod-ingestor
@@ -199,7 +199,7 @@ function main()
     }
 
     if ($realmToAggregate !== false
-        && !in_array($realmToAggregate, ['job', 'cloud', 'storage'])) {
+        && !in_array($realmToAggregate, ['job', 'cloud', 'storage', 'cloudresourcespecs'])) {
         $logger->crit("Invalid realm '$value' specified for aggregation");
         exit(1);
     }
@@ -279,6 +279,10 @@ function main()
                 if ($datatypeValue == 'storage') {
                     $dwi->ingestStorageData();
                 }
+
+                if ($datatypeValue == 'cloudresourcespecs') {
+                    $dwi->ingestCloudResourceSpecs();
+                }
             }
 
             $aclstatus = 0;
@@ -311,6 +315,9 @@ function main()
             if ($realmToAggregate == 'storage' || $realmToAggregate === false) {
                 $dwi->aggregateStorageData($lastModifiedStartDate);
             }
+            if ($realmToAggregate == 'cloudresourcespecs' || $realmToAggregate === false) {
+                $dwi->aggregateCloudResourceSpecs($lastModifiedStartDate);
+            }
         } catch (Exception $e) {
             $logger->crit(array(
                 'message'    => 'Aggregation failed: ' . $e->getMessage(),

diff --git a/classes/ETL/Ingestor/StateReconstructorTransformIngestor.php b/classes/ETL/Ingestor/StateReconstructorTransformIngestor.php
diff --git a/classes/OpenXdmod/DataWarehouseInitializer.php b/classes/OpenXdmod/DataWarehouseInitializer.php
@@ -246,6 +246,8 @@ public function ingestCloudDataOpenStack()
                 ),
                 $this->logger
             );
+
+            $this->ingestCloudResourceSpecs();
         }
         catch( Exception $e ){
             if( $e->getCode() == 1146 ){
@@ -257,6 +259,23 @@ public function ingestCloudDataOpenStack()
         }
     }
 
+    /**
+     * Ingesting the cloud resource specifications from the raw_resource_specs table
+     * in modw_cloud to the cloud_resource_specs table
+     */
+    public function ingestCloudResourceSpecs(){
+        if( !$this->isRealmEnabled('Cloud') ){
+            $this->logger->debug('Cloud realm not enabled, not ingesting');
+            return;
+        }
+
+        $this->logger->notice('Ingesting cloud resource specs');
+        Utilities::runEtlPipeline(
+            array('ingest-cloud-resource-specs'),
+            $this->logger
+        );
+    }
+
     /**
      * Extracting cloud log data from the generic_raw_events table. If the raw
      * tables do not exist then catch the resulting exception and display a message
@@ -275,6 +294,8 @@ public function ingestCloudDataGeneric()
                 array('jobs-cloud-common', 'jobs-cloud-import-users-generic', 'jobs-cloud-extract-generic'),
                 $this->logger
             );
+
+            $this->ingestCloudResourceSpecs();
         }
         catch( Exception $e ){
             if( $e->getCode() == 1146 ){
@@ -330,11 +351,34 @@ public function aggregateCloudData($lastModifiedStartDate)
             array('last-modified-start-date' => $lastModifiedStartDate)
         );
 
+        $this->aggregateCloudResourceSpecs($lastModifiedStartDate);
+
         $filterListBuilder = new FilterListBuilder();
         $filterListBuilder->setLogger($this->logger);
         $filterListBuilder->buildRealmLists('Cloud');
     }
 
+    /**
+     * Aggregated cloud resource specifications for use with utilization statistic
+     * in the cloud realm. If the cloud realm is not enabled then do nothing
+     *
+     * @param string $lastModifiedStartDate Aggregate data ingested on or after
+     *     this date.
+     */
+    public function aggregateCloudResourceSpecs($lastModifiedStartDate){
+        if( !$this->isRealmEnabled('Cloud') ){
+            $this->logger->debug('Cloud realm not enabled, not aggregating');
+            return;
+        }
+
+        $this->logger->notice('Aggregating Cloud Resource Specification data');
+        Utilities::runEtlPipeline(
+            array('aggregate-cloud-resource-specs'),
+            $this->logger,
+            array('last-modified-start-date' => $lastModifiedStartDate)
+        );
+    }
+
     /**
      * Aggregate storage data.
      *