Add new file/folder survey options

open-metadata-implementation/adapters/open-connectors/data-store-connectors/jdbc-resource-connector/src/main/java/org/odpi/openmetadata/adapters/connectors/resource/jdbc/JDBCResourceConnector.java

@@ -178,6 +178,17 @@ public synchronized void start() throws ConnectorCheckedException
     }
 
 
+    /**
+     * Return the name of the database.
+     *
+     * @return string name
+     */
+    public String getDatabaseName()
+    {
+        return jdbcDatabaseName;
+    }
+
+
     /**
      * Return the DataSource implementation for this database.  This supports creating the connection to the database.
      *

open-metadata-implementation/adapters/open-connectors/file-survey-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/surveyaction/extractors/FileStatsExtractor.java

@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright Contributors to the ODPi Egeria project. */
+
+package org.odpi.openmetadata.adapters.connectors.surveyaction.extractors;
+
+import org.odpi.openmetadata.adapters.connectors.surveyaction.surveyfile.SurveyFileAnnotationType;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.InvalidParameterException;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.PropertyServerException;
+import org.odpi.openmetadata.frameworks.connectors.ffdc.UserNotAuthorizedException;
+import org.odpi.openmetadata.frameworks.governanceaction.fileclassifier.FileClassification;
+import org.odpi.openmetadata.frameworks.governanceaction.fileclassifier.FileClassifier;
+import org.odpi.openmetadata.frameworks.surveyaction.SurveyActionServiceConnector;
+import org.odpi.openmetadata.frameworks.surveyaction.measurements.FileMeasurement;
+import org.odpi.openmetadata.frameworks.surveyaction.measurements.FileMetric;
+import org.odpi.openmetadata.frameworks.surveyaction.properties.Annotation;
+import org.odpi.openmetadata.frameworks.surveyaction.properties.ResourcePhysicalStatusAnnotation;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * FileStatsExtractor uses the FileClassifier to extract the stats about a file and then convert them into a list of
+ * annotation.
+ */
+public class FileStatsExtractor
+{
+    final private File file;
+    final private SurveyActionServiceConnector surveyActionServiceConnector;
+    final private FileClassifier fileClassifier;
+
+
+    /**
+     * Constructor captures the file that is to be analysed.
+     *
+     * @param file file to analyse
+     */
+    public FileStatsExtractor(File                         file,
+                              FileClassifier               fileClassifier,
+                              SurveyActionServiceConnector surveyActionServiceConnector)
+    {
+        this.file = file;
+        this.fileClassifier = fileClassifier;
+        this.surveyActionServiceConnector = surveyActionServiceConnector;
+    }
+
+
+    public Annotation getAnnotation() throws InvalidParameterException,
+                                             PropertyServerException,
+                                             UserNotAuthorizedException,
+                                             IOException
+    {
+        FileClassification fileClassification = fileClassifier.classifyFile(file);
+
+        ResourcePhysicalStatusAnnotation measurementAnnotation = new ResourcePhysicalStatusAnnotation();
+
+        measurementAnnotation.setAnnotationType(SurveyFileAnnotationType.MEASUREMENTS.getName());
+        measurementAnnotation.setSummary(SurveyFileAnnotationType.MEASUREMENTS.getSummary());
+        measurementAnnotation.setExplanation(SurveyFileAnnotationType.MEASUREMENTS.getExplanation());
+        measurementAnnotation.setCreateTime(fileClassification.getCreationTime());
+        measurementAnnotation.setModifiedTime(fileClassification.getLastModifiedTime());
+        measurementAnnotation.setLastAccessedTime(fileClassification.getLastAccessedTime());
+        measurementAnnotation.setSize(fileClassification.getFileSize());
+
+        Map<String, String> dataSourceProperties = new HashMap<>();
+
+        dataSourceProperties.put(FileMetric.FILE_NAME.displayName, fileClassification.getFileName());
+        dataSourceProperties.put(FileMetric.PATH_NAME.displayName, fileClassification.getPathName());
+        dataSourceProperties.put(FileMetric.FILE_EXTENSION.displayName, fileClassification.getFileExtension());
+        dataSourceProperties.put(FileMetric.FILE_TYPE.displayName, fileClassification.getFileType());
+        dataSourceProperties.put(FileMetric.DEPLOYED_IMPLEMENTATION_TYPE.displayName, fileClassification.getDeployedImplementationType());
+        dataSourceProperties.put(FileMetric.ENCODING.displayName, fileClassification.getEncoding());
+        dataSourceProperties.put(FileMetric.ASSET_TYPE_NAME.displayName, fileClassification.getAssetTypeName());
+        dataSourceProperties.put(FileMetric.CAN_READ.displayName, Boolean.toString(fileClassification.isCanRead()));
+        dataSourceProperties.put(FileMetric.CAN_WRITE.displayName, Boolean.toString(fileClassification.isCanWrite()));
+        dataSourceProperties.put(FileMetric.CAN_EXECUTE.displayName, Boolean.toString(fileClassification.isCanExecute()));
+        dataSourceProperties.put(FileMetric.IS_SYM_LINK.displayName, Boolean.toString(fileClassification.isSymLink()));
+        dataSourceProperties.put(FileMetric.IS_HIDDEN.displayName, Boolean.toString(fileClassification.isHidden()));
+        dataSourceProperties.put(FileMetric.FILE_SIZE.displayName, Long.toString(fileClassification.getFileSize()));
+        if (fileClassification.getCreationTime() != null)
+        {
+            dataSourceProperties.put(FileMetric.CREATION_TIME.displayName, fileClassification.getCreationTime().toString());
+        }
+        if (fileClassification.getLastModifiedTime() != null)
+        {
+            dataSourceProperties.put(FileMetric.LAST_MODIFIED_TIME.displayName, fileClassification.getLastModifiedTime().toString());
+        }
+        if (fileClassification.getLastAccessedTime() != null)
+        {
+            dataSourceProperties.put(FileMetric.LAST_ACCESSED_TIME.displayName, fileClassification.getLastAccessedTime().toString());
+        }
+
+        measurementAnnotation.setResourceProperties(dataSourceProperties);
+
+        FileMeasurement fileMeasurement = new FileMeasurement();
+
+        fileMeasurement.setFileName(fileClassification.getFileName());
+        fileMeasurement.setPathName(fileClassification.getPathName());
+        fileMeasurement.setFileExtension(fileClassification.getFileExtension());
+        fileMeasurement.setFileType(fileClassification.getFileType());
+        fileMeasurement.setDeployedImplementationType(fileClassification.getDeployedImplementationType());
+        fileMeasurement.setEncoding(fileClassification.getEncoding());
+        fileMeasurement.setFileName(fileClassification.getFileName());
+        fileMeasurement.setAssetTypeName(fileClassification.getAssetTypeName());
+        fileMeasurement.setCanRead(fileClassification.isCanRead());
+        fileMeasurement.setCanWrite(fileClassification.isCanWrite());
+        fileMeasurement.setCanExecute(fileClassification.isCanExecute());
+        fileMeasurement.setSymLink(fileClassification.isSymLink());
+        fileMeasurement.setHidden(fileClassification.isHidden());
+        fileMeasurement.setCreationTime(fileClassification.getCreationTime());
+        fileMeasurement.setLastModifiedTime(fileClassification.getLastModifiedTime());
+        fileMeasurement.setLastAccessedTime(fileClassification.getLastAccessedTime());
+        fileMeasurement.setFileSize(fileClassification.getFileSize());
+
+        measurementAnnotation.setJsonProperties(surveyActionServiceConnector.getJSONProperties(fileMeasurement));
+
+        return measurementAnnotation;
+    }
+}

open-metadata-implementation/adapters/open-connectors/file-survey-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/surveyaction/ffdc/SurveyServiceAuditCode.java

@@ -51,11 +51,11 @@ public enum SurveyServiceAuditCode implements AuditLogMessageSet
 
 
     /**
-     * SURVEY-ACTION-SERVICE-0004 - The survey action service {0} is surveying the {1} folder (directory)
+     * SURVEY-ACTION-SERVICE-0004 - The survey action service {0} is surveying the {1} folder (directory) with an analysis level of {2}
      */
     SURVEYING_FOLDER("SURVEY-ACTION-SERVICE-0004",
                      AuditLogRecordSeverityLevel.INFO,
-                     "The survey action service {0} is surveying the {1} folder (directory)",
+                     "The survey action service {0} is surveying the {1} folder (directory) with an analysis level of {2}",
                      "This message shows that the starting folder to survey.",
                      "No specific action is required.  This message is marking the start of the survey process."),
 

open-metadata-implementation/adapters/open-connectors/file-survey-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/surveyaction/surveycsv/CSVSurveyService.java

@@ -3,7 +3,7 @@
 package org.odpi.openmetadata.adapters.connectors.surveyaction.surveycsv;
 
 import org.odpi.openmetadata.adapters.connectors.datastore.csvfile.CSVFileStoreConnector;
-import org.odpi.openmetadata.adapters.connectors.surveyaction.surveyfile.FileMetric;
+import org.odpi.openmetadata.frameworks.surveyaction.measurements.FileMetric;
 import org.odpi.openmetadata.adapters.connectors.surveyaction.surveyfile.SurveyFileAnnotationType;
 import org.odpi.openmetadata.frameworks.connectors.Connector;
 import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectorCheckedException;
@@ -229,19 +229,19 @@ else if (! propertyHelper.isTypeOf(assetUniverse, OpenMetadataType.CSV_FILE.type
 
             Map<String, String> dataSourceProperties = new HashMap<>();
 
-            dataSourceProperties.put(FileMetric.FILE_NAME.name, fileClassification.getFileName());
-            dataSourceProperties.put(FileMetric.PATH_NAME.name, fileClassification.getPathName());
-            dataSourceProperties.put(FileMetric.FILE_EXTENSION.name, fileClassification.getFileExtension());
-            dataSourceProperties.put(FileMetric.FILE_TYPE.name, fileClassification.getFileType());
-            dataSourceProperties.put(FileMetric.DEPLOYED_IMPLEMENTATION_TYPE.name, fileClassification.getDeployedImplementationType());
-            dataSourceProperties.put(FileMetric.ENCODING.name, fileClassification.getEncoding());
-            dataSourceProperties.put(FileMetric.ASSET_TYPE_NAME.name, fileClassification.getAssetTypeName());
-            dataSourceProperties.put(FileMetric.CAN_READ.name, Boolean.toString(fileClassification.isCanRead()));
-            dataSourceProperties.put(FileMetric.CAN_WRITE.name, Boolean.toString(fileClassification.isCanWrite()));
-            dataSourceProperties.put(FileMetric.CAN_EXECUTE.name, Boolean.toString(fileClassification.isCanExecute()));
-            dataSourceProperties.put(FileMetric.IS_SYM_LINK.name, Boolean.toString(fileClassification.isSymLink()));
-            dataSourceProperties.put(FileMetric.IS_HIDDEN.name, Boolean.toString(fileClassification.isHidden()));
-            dataSourceProperties.put(FileMetric.RECORD_COUNT.name, Long.toString(recordCount));
+            dataSourceProperties.put(FileMetric.FILE_NAME.displayName, fileClassification.getFileName());
+            dataSourceProperties.put(FileMetric.PATH_NAME.displayName, fileClassification.getPathName());
+            dataSourceProperties.put(FileMetric.FILE_EXTENSION.displayName, fileClassification.getFileExtension());
+            dataSourceProperties.put(FileMetric.FILE_TYPE.displayName, fileClassification.getFileType());
+            dataSourceProperties.put(FileMetric.DEPLOYED_IMPLEMENTATION_TYPE.displayName, fileClassification.getDeployedImplementationType());
+            dataSourceProperties.put(FileMetric.ENCODING.displayName, fileClassification.getEncoding());
+            dataSourceProperties.put(FileMetric.ASSET_TYPE_NAME.displayName, fileClassification.getAssetTypeName());
+            dataSourceProperties.put(FileMetric.CAN_READ.displayName, Boolean.toString(fileClassification.isCanRead()));
+            dataSourceProperties.put(FileMetric.CAN_WRITE.displayName, Boolean.toString(fileClassification.isCanWrite()));
+            dataSourceProperties.put(FileMetric.CAN_EXECUTE.displayName, Boolean.toString(fileClassification.isCanExecute()));
+            dataSourceProperties.put(FileMetric.IS_SYM_LINK.displayName, Boolean.toString(fileClassification.isSymLink()));
+            dataSourceProperties.put(FileMetric.IS_HIDDEN.displayName, Boolean.toString(fileClassification.isHidden()));
+            dataSourceProperties.put(FileMetric.RECORD_COUNT.displayName, Long.toString(recordCount));
 
             measurementAnnotation.setResourceProperties(dataSourceProperties);
 

open-metadata-implementation/adapters/open-connectors/file-survey-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/surveyaction/surveyfile/FileSurveyService.java

@@ -3,6 +3,7 @@
 package org.odpi.openmetadata.adapters.connectors.surveyaction.surveyfile;
 
 import org.odpi.openmetadata.adapters.connectors.datastore.basicfile.BasicFileStore;
+import org.odpi.openmetadata.adapters.connectors.surveyaction.extractors.FileStatsExtractor;
 import org.odpi.openmetadata.frameworks.connectors.Connector;
 import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectorCheckedException;
 import org.odpi.openmetadata.frameworks.connectors.properties.AssetUniverse;
@@ -14,12 +15,12 @@
 import org.odpi.openmetadata.frameworks.surveyaction.SurveyActionServiceConnector;
 import org.odpi.openmetadata.frameworks.surveyaction.SurveyAssetStore;
 import org.odpi.openmetadata.frameworks.surveyaction.controls.AnalysisStep;
+import org.odpi.openmetadata.frameworks.surveyaction.measurements.FileMeasurement;
+import org.odpi.openmetadata.frameworks.surveyaction.measurements.FileMetric;
+import org.odpi.openmetadata.frameworks.surveyaction.properties.Annotation;
 import org.odpi.openmetadata.frameworks.surveyaction.properties.ResourcePhysicalStatusAnnotation;
 
 import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -81,41 +82,18 @@ else if (! propertyHelper.isTypeOf(assetUniverse, OpenMetadataType.DATA_FILE.typ
 
             File file = assetConnector.getFile();
 
-            BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
-
             annotationStore.setAnalysisStep(AnalysisStep.MEASURE_RESOURCE.getName());
 
-            FileClassifier fileClassifier = surveyContext.getFileClassifier();
-            FileClassification fileClassification = fileClassifier.classifyFile(file);
-
-            ResourcePhysicalStatusAnnotation measurementAnnotation = new ResourcePhysicalStatusAnnotation();
-
-            measurementAnnotation.setAnnotationType(SurveyFileAnnotationType.MEASUREMENTS.getName());
-            measurementAnnotation.setSummary(SurveyFileAnnotationType.MEASUREMENTS.getSummary());
-            measurementAnnotation.setExplanation(SurveyFileAnnotationType.MEASUREMENTS.getExplanation());
-            measurementAnnotation.setCreateTime(fileClassification.getCreationTime());
-            measurementAnnotation.setModifiedTime(fileClassification.getLastModifiedTime());
-            measurementAnnotation.setLastAccessedTime(fileClassification.getLastAccessedTime());
-            measurementAnnotation.setSize(assetConnector.getFileLength());
-
-            Map<String, String> dataSourceProperties = new HashMap<>();
-
-            dataSourceProperties.put(FileMetric.FILE_NAME.name, fileClassification.getFileName());
-            dataSourceProperties.put(FileMetric.PATH_NAME.name, fileClassification.getPathName());
-            dataSourceProperties.put(FileMetric.FILE_EXTENSION.name, fileClassification.getFileExtension());
-            dataSourceProperties.put(FileMetric.FILE_TYPE.name, fileClassification.getFileType());
-            dataSourceProperties.put(FileMetric.DEPLOYED_IMPLEMENTATION_TYPE.name, fileClassification.getDeployedImplementationType());
-            dataSourceProperties.put(FileMetric.ENCODING.name, fileClassification.getEncoding());
-            dataSourceProperties.put(FileMetric.ASSET_TYPE_NAME.name, fileClassification.getAssetTypeName());
-            dataSourceProperties.put(FileMetric.CAN_READ.name, Boolean.toString(fileClassification.isCanRead()));
-            dataSourceProperties.put(FileMetric.CAN_WRITE.name, Boolean.toString(fileClassification.isCanWrite()));
-            dataSourceProperties.put(FileMetric.CAN_EXECUTE.name, Boolean.toString(fileClassification.isCanExecute()));
-            dataSourceProperties.put(FileMetric.IS_SYM_LINK.name, Boolean.toString(fileClassification.isSymLink()));
-            dataSourceProperties.put(FileMetric.IS_HIDDEN.name, Boolean.toString(fileClassification.isHidden()));
-
-            measurementAnnotation.setResourceProperties(dataSourceProperties);
-
-            annotationStore.addAnnotation(measurementAnnotation, surveyContext.getAssetGUID());
+            FileStatsExtractor fileStatsExtractor = new FileStatsExtractor(file,
+                                                                           surveyContext.getFileClassifier(),
+                                                                           this);
+
+            Annotation measurementAnnotation = fileStatsExtractor.getAnnotation();
+
+            if (measurementAnnotation != null)
+            {
+                annotationStore.addAnnotation(measurementAnnotation, surveyContext.getAssetGUID());
+            }
         }
         catch (ConnectorCheckedException error)
         {