diff --git a/.arc-validate-results/arc_specification@2.0.0/badge.svg b/.arc-validate-results/arc_specification@2.0.0/badge.svg new file mode 100644 index 0000000..ec7c9d9 --- /dev/null +++ b/.arc-validate-results/arc_specification@2.0.0/badge.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + arc_specification + arc_specification + + + 8 Critical Errors + 8 Critical Errors + + diff --git a/.arc-validate-results/arc_specification@2.0.0/validation_report.xml b/.arc-validate-results/arc_specification@2.0.0/validation_report.xml new file mode 100644 index 0000000..10cef44 --- /dev/null +++ b/.arc-validate-results/arc_specification@2.0.0/validation_report.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/.arc-validate-results/arc_specification@2.0.0/validation_summary.json b/.arc-validate-results/arc_specification@2.0.0/validation_summary.json new file mode 100644 index 0000000..5d80d17 --- /dev/null +++ b/.arc-validate-results/arc_specification@2.0.0/validation_summary.json @@ -0,0 +1 @@ +{"Critical":{"HasFailures":true,"Total":17,"Passed":9,"Failed":8,"Errored":0},"NonCritical":{"HasFailures":false,"Total":0,"Passed":0,"Failed":0,"Errored":0},"ValidationPackage":{"Name":"arc_specification","Version":"2.0.0","Summary":"Validate whether an ARC conforms to Specification V2.0.0-draft","Description":"Validate whether an ARC conforms to Specification V2.0.0-draft. See the relevant spec at https://github.com/nfdi4plants/ARC-specification/blob/v2.0.0/ARC%20specification.md"}} \ No newline at end of file diff --git a/src/ARCExpect/ARCExpect.fsproj b/src/ARCExpect/ARCExpect.fsproj index 0094523..09f015a 100644 --- a/src/ARCExpect/ARCExpect.fsproj +++ b/src/ARCExpect/ARCExpect.fsproj @@ -1,13 +1,12 @@ - netstandard2.0 + net8.0 true - @@ -18,13 +17,12 @@ + + + + + - - - - - - @@ -36,5 +34,7 @@ + + \ No newline at end of file diff --git a/src/ARCExpect/ARCValidationPackage.fs b/src/ARCExpect/ARCValidationPackage.fs new file mode 100644 index 0000000..fddf4cc --- /dev/null +++ b/src/ARCExpect/ARCValidationPackage.fs @@ -0,0 +1,53 @@ +namespace ARCExpect + +open AVPRIndex +open Expecto + +type ARCValidationPackage = + { + Metadata: ValidationPackageMetadata + CriticalValidationCases: Test + NonCriticalValidationCases: Test + CQCHookEndpoint: string option + } with + static member create ( + metadata: ValidationPackageMetadata, + criticalValidationCases: Test, + nonCriticalValidationCases: Test, + ?CQCHookEndpoint: string + ) = + { + Metadata = metadata + CriticalValidationCases = criticalValidationCases + NonCriticalValidationCases = nonCriticalValidationCases + CQCHookEndpoint = CQCHookEndpoint + } + + static member create ( + metadata: ValidationPackageMetadata, + criticalValidationCasesList: Test list, + nonCriticalValidationCasesList: Test list, + ?CQCHookEndpoint: string + ) = + let criticalCases = testList "Critical" criticalValidationCasesList + let nonCriticalCases = testList "NonCritical" nonCriticalValidationCasesList + + ARCValidationPackage.create( + metadata = metadata, + criticalValidationCases = criticalCases, + nonCriticalValidationCases = nonCriticalCases, + ?CQCHookEndpoint = CQCHookEndpoint + ) + + static member create ( + metadata: ValidationPackageMetadata, + ?CriticalValidationCasesList: Test list, + ?NonCriticalValidationCasesList: Test list, + ?CQCHookEndpoint: string + ) = + ARCValidationPackage.create( + metadata = metadata, + criticalValidationCasesList = defaultArg CriticalValidationCasesList [], + nonCriticalValidationCasesList = defaultArg NonCriticalValidationCasesList [], + ?CQCHookEndpoint = CQCHookEndpoint + ) \ No newline at end of file diff --git a/src/ARCExpect/BadgeCreation.fs b/src/ARCExpect/BadgeCreation.fs index ee71fca..eedad9d 100644 --- a/src/ARCExpect/BadgeCreation.fs +++ b/src/ARCExpect/BadgeCreation.fs @@ -36,3 +36,50 @@ type BadgeCreation = value = testResults.passed.Length, ValueSuffix = $"/{max}" ) + + + static member ofValidationSummary( + labelText: string, + ?ValueSuffix: string, + ?Thresholds: Map, + ?DefaultColor: Color + ) = + + fun (validationSummary: ValidationSummary) -> + + let total = validationSummary.Critical.Total + validationSummary.NonCritical.Total + + let totalPassed = validationSummary.Critical.Passed + validationSummary.NonCritical.Passed + + let criticalFailedOrErrored = validationSummary.Critical.Failed + validationSummary.Critical.Errored + + if validationSummary.Critical.HasFailures then + + Badge( + label = labelText, + defaultColor = Color.RED, + value = criticalFailedOrErrored, + ValueSuffix = $" Critical Errors" + ) + + else + + let thresholds = + Thresholds + |> Option.defaultValue (Map([ + 0, Color.RED + total/2, Color.ORANGE_2 + total, Color.GREEN + ])) + + let valueSuffix = + ValueSuffix + |> Option.defaultValue $"/{total}" + + Badge( + label = labelText, + defaultColor = (DefaultColor |> Option.defaultValue (Color.fromString Defaults.DEFAULT_COLOR) ), + Thresholds = thresholds, + value = totalPassed, + ValueSuffix = valueSuffix + ) \ No newline at end of file diff --git a/src/ARCExpect/Configs/ARCConfig.fs b/src/ARCExpect/Configs/ARCConfig.fs deleted file mode 100644 index 99f54d9..0000000 --- a/src/ARCExpect/Configs/ARCConfig.fs +++ /dev/null @@ -1,56 +0,0 @@ -namespace ARCExpect.Configs -open ARCExpect -open System.IO - -type ARCConfig(pathConfig : PathConfig) = - - new(arcRootPath : string) = - let pathConfig = new PathConfig(arcRootPath) - ARCConfig(pathConfig) - - member val PathConfig = pathConfig with get, set - - // Investigation (might be worth to separate into mutiple configs) - - member this.InvestigationWorkbook = () - - member this.InvestigationWorksheet = () - - member this.InvestigationPathCvP = () - - member this.InvestigationTokens = () - - member this.InvestigationContainers = () - - member this.InvestigationContainer = () - - member this.InvestigationStudies = () - - member this.InvestigationContactsContainer = () - - // Study (might be worth to separate into mutiple configs) - - member this.StudyPathsAndIds = () - - member this.StudyFolders = () - - member this.StudyFilesAndIds = () - - member this.StudyAnnotationTables = () - - member this.StudyRawOrDerivedDataPaths = () - - // Assay (might be worth to separate into mutiple configs) - - member this.AssayFolders = () - - member this.AssayFilesAndIds = () - - member this.AssayAnnotationTables = () - - member this.AssayRawOrDerivedDataPaths = () - - // Composite - - //member this.DataPaths = Seq.append this.StudyRawOrDerivedDataPaths this.AssayRawOrDerivedDataPaths - member this.DataPaths = () diff --git a/src/ARCExpect/Configs/PathConfig.fs b/src/ARCExpect/Configs/PathConfig.fs deleted file mode 100644 index 8e3a081..0000000 --- a/src/ARCExpect/Configs/PathConfig.fs +++ /dev/null @@ -1,56 +0,0 @@ -/// Paths to folders and files that must/should exist in an ARC. -namespace ARCExpect.Configs - -open System.IO - -type PathConfig(arcRootPath: string) = - - // settable root path. All other properties are explicit properties meaning that they get evaluated everytime they are accessed -> they are always changed accordingly to the root path. - // Useful for setting correct root path via CLI args or defaulting to another value. - member val ARCRootPath = arcRootPath with get, set - - // - member this.StudiesPath = Path.Combine(this.ARCRootPath, "studies") - member this.AssaysPath = Path.Combine(this.ARCRootPath, "assays") - member this.RunsPath = Path.Combine(this.ARCRootPath, "runs") - member this.WorkflowsPath = Path.Combine(this.ARCRootPath, "workflows") - member this.InvestigationPath = Path.Combine(this.ARCRootPath, "isa.investigation.xlsx") - - // - member this.DotARCFolderPath = Path.Combine(this.ARCRootPath, ".arc") - - // - member this.GitFolderPath = Path.Combine(this.ARCRootPath, ".git") - member this.ConfigPath = Path.Combine(this.GitFolderPath, "config") - member this.DescriptionPath = Path.Combine(this.GitFolderPath, "description") - member this.HeadPath = Path.Combine(this.GitFolderPath, "HEAD") - - // - member this.ObjectsPath = Path.Combine(this.GitFolderPath, "objects") - member this.ObjectsInfoPath = Path.Combine(this.ObjectsPath, "info") - member this.ObjectsPackPath = Path.Combine(this.ObjectsPath, "pack") - - // - member this.InfoPath = Path.Combine(this.GitFolderPath, "info") - member this.ExcludePath = Path.Combine(this.InfoPath, "exclude") - - // - member this.RefsPath = Path.Combine(this.GitFolderPath, "refs") - member this.RefsHeadsPath = Path.Combine(this.RefsPath, "heads") - member this.RefsTagsPath = Path.Combine(this.RefsPath, "tags") - - // - member this.HooksPath = Path.Combine(this.GitFolderPath, "hooks") - member this.ApplyPatchPath = Path.Combine(this.HooksPath, "applypatch-msg.sample") - member this.CommitSamplePath = Path.Combine(this.HooksPath, "commit-msg.sample") - member this.FsmonitorPath = Path.Combine(this.HooksPath, "fsmonitor-watchman.sample") - member this.PostUpdatePath = Path.Combine(this.HooksPath, "post-update.sample") - member this.PreApplyPatchPath = Path.Combine(this.HooksPath, "pre-applypatch.sample") - member this.PreCommitPath = Path.Combine(this.HooksPath, "pre-commit.sample") - member this.PreMergeCommitPath = Path.Combine(this.HooksPath, "pre-merge-commit.sample") - member this.PrePushPath = Path.Combine(this.HooksPath, "pre-push.sample") - member this.PreRebasePath = Path.Combine(this.HooksPath, "pre-rebase.sample") - member this.PreReceivePath = Path.Combine(this.HooksPath, "pre-receive.sample") - member this.PrepareCommitPath = Path.Combine(this.HooksPath, "prepare-commit-msg.sample") - member this.PushToCheckoutPath = Path.Combine(this.HooksPath, "push-to-checkout.sample") - member this.UpdatePath = Path.Combine(this.HooksPath, "update.sample") \ No newline at end of file diff --git a/src/ARCExpect/InternalUtils.fs b/src/ARCExpect/InternalUtils.fs index 52bffce..872fb22 100644 --- a/src/ARCExpect/InternalUtils.fs +++ b/src/ARCExpect/InternalUtils.fs @@ -3,6 +3,8 @@ open System open Graphoscope +open System.Text.Json +open System.Text.Json.Serialization //// this is needed to allow ValidatorTests project to access internal modules @@ -20,6 +22,13 @@ module InternalUtils = open OBO.NET open ControlledVocabulary + /// internal json options for better F# type support in serialization (mainly for Options) + module JsonOptions = + let options = + JsonFSharpOptions.Default() + .WithSkippableOptionFields() // if option is none, do not include a property, but include it if option is some. + .ToJsonSerializerOptions() + module String = diff --git a/src/ARCExpect/SpecificationValidation/SpecificationSelection.fs b/src/ARCExpect/SpecificationValidation/SpecificationSelection.fs new file mode 100644 index 0000000..1797a5b --- /dev/null +++ b/src/ARCExpect/SpecificationValidation/SpecificationSelection.fs @@ -0,0 +1,20 @@ +namespace ARCExpect.SpecificationValidation + +module SpecificationSelection = + + open ARCExpect + + let [] latest = "2.0.0-draft" + + let internal specMap = + [ + "2.0.0-draft" , SpecificationValidation.V2_0_0_Draft.validationCases + ] |> Map.ofSeq + + let tryGetValidationCasesForSpecificationVersion (specVersion: string) (path: string) = + + let specVersion = if specVersion = "latest" then latest else specVersion + + match specMap.TryFind specVersion with + | Some generator -> specVersion, generator path + | None -> failwithf "No validation cases found for specification version '%s'" specVersion \ No newline at end of file diff --git a/src/ARCExpect/ARCSpecification.fs b/src/ARCExpect/SpecificationValidation/V2_0_0_Draft.fs similarity index 98% rename from src/ARCExpect/ARCSpecification.fs rename to src/ARCExpect/SpecificationValidation/V2_0_0_Draft.fs index 6178e57..85d2d32 100644 --- a/src/ARCExpect/ARCSpecification.fs +++ b/src/ARCExpect/SpecificationValidation/V2_0_0_Draft.fs @@ -1,11 +1,12 @@ -namespace ARCExpect.ARCSpecification +namespace ARCExpect.SpecificationValidation open ControlledVocabulary open ARCTokenization open ARCExpect open Expecto +open AVPRIndex -module V2_Draft = +module V2_0_0_Draft = module MustHaveTerms = let investigationTerms = @@ -268,7 +269,7 @@ module V2_Draft = /// /// The code uses the ARCExpect module and various functions from the Validate module to perform these checks. let cases = - testList "cases" [ + [ //Check for investigation ARCExpect.validationCase (TestID.Name "Arc contains investigation file") { @@ -550,4 +551,14 @@ module V2_Draft = } ] - cases \ No newline at end of file + ARCValidationPackage.create( + metadata = ValidationPackageMetadata.create( + name = "arc_specification", + summary = "Validate whether an ARC conforms to Specification V2.0.0-draft", + description = "Validate whether an ARC conforms to Specification V2.0.0-draft. See the relevant spec at https://github.com/nfdi4plants/ARC-specification/blob/v2.0.0/ARC%20specification.md", + majorVersion = 2, + minorVersion = 0, + patchVersion = 0 + ), + CriticalValidationCasesList = cases + ) \ No newline at end of file diff --git a/src/ARCExpect/TestGeneration/Critical/ARCFileSystem.fs b/src/ARCExpect/TestGeneration/Critical/ARCFileSystem.fs deleted file mode 100644 index 997ff8b..0000000 --- a/src/ARCExpect/TestGeneration/Critical/ARCFileSystem.fs +++ /dev/null @@ -1,31 +0,0 @@ -namespace ARCExpect.TestGeneration.Critical.ARC - -open ARCExpect -open ControlledVocabulary -open ARCTokenization -open CvParamExtensions -open ARCExpect.Configs - -open ARCTokenization -open ARCTokenization.StructuralOntology - -module FileSystem = - - open Expecto - open FSharpAux - - let generateARCFileSystemTests (rootPath: string) = - - let relativeDirectoryPaths = FileSystem.parseRelativeDirectoryPaths rootPath - let relativeFilePaths = FileSystem.parseRelativeFilePaths rootPath - - //let pathConfig = arcConfig.PathConfig - - // we need a structuralontology here as well (e.g. terms for 'Investigation File', 'Study File') - testList "FileSystem" [ - testList "Investigation" [ - validationCase (TestID.Name "Investigation File") { - relativeFilePaths |> Validate.ParamCollection.ContainsParamWithValue "isa.investigation.xlsx" - } - ] - ] \ No newline at end of file diff --git a/src/ARCExpect/TestGeneration/Critical/ARCISA.fs b/src/ARCExpect/TestGeneration/Critical/ARCISA.fs deleted file mode 100644 index 2ca9c28..0000000 --- a/src/ARCExpect/TestGeneration/Critical/ARCISA.fs +++ /dev/null @@ -1,34 +0,0 @@ -namespace ARCExpect.TestGeneration.Critical.ARC - -open ARCExpect -open ControlledVocabulary -open ARCTokenization -open CvParamExtensions -open ARCExpect.Configs - -open ARCTokenization -open ARCTokenization.StructuralOntology - -module ISA = - - open Expecto - open FSharpAux - - let generateISATests (tokens: IParam list) = - - let cvParams = tokens |> List.choose Param.tryCvParam - - //let pathConfig = arcConfig.PathConfig - - testList INVMSO.``Investigation Metadata``.key.Name [ - testList INVMSO.``Investigation Metadata``.INVESTIGATION.key.Name [ - validationCase (TestID.Name INVMSO.``Investigation Metadata``.INVESTIGATION.``Investigation Title``.Name) { - cvParams - |> Validate.ParamCollection.ContainsParamWithTerm INVMSO.``Investigation Metadata``.INVESTIGATION.``Investigation Title`` - } - validationCase (TestID.Name INVMSO.``Investigation Metadata``.INVESTIGATION.``Investigation Description``.Name) { - cvParams - |> Validate.ParamCollection.ContainsParamWithTerm INVMSO.``Investigation Metadata``.INVESTIGATION.``Investigation Description`` - } - ] - ] \ No newline at end of file diff --git a/src/ARCExpect/TestGeneration/NonCritical/ARCISA.fs b/src/ARCExpect/TestGeneration/NonCritical/ARCISA.fs deleted file mode 100644 index fecc753..0000000 --- a/src/ARCExpect/TestGeneration/NonCritical/ARCISA.fs +++ /dev/null @@ -1,27 +0,0 @@ -namespace ARCExpect.TestGeneration.NonCritical.ARC - -open ARCExpect -open ControlledVocabulary -open ARCTokenization -open CvParamExtensions -open ARCExpect.Configs - -open ARCTokenization -open ARCTokenization.StructuralOntology - - -module ISA = - - open Expecto - open FSharpAux - //open System.IO - - let generateISATests (tokens: IParam list) = - - let cvParams = tokens |> List.choose Param.tryCvParam - - //let pathConfig = arcConfig.PathConfig - - testList INVMSO.``Investigation Metadata``.key.Name [ - - ] \ No newline at end of file diff --git a/src/ARCExpect/TopLevelAPI.fs b/src/ARCExpect/TopLevelAPI.fs index d9ad339..bc69335 100644 --- a/src/ARCExpect/TopLevelAPI.fs +++ b/src/ARCExpect/TopLevelAPI.fs @@ -3,17 +3,100 @@ open AnyBadge.NET open Expecto open System.IO +open AVPRIndex -type Execute = +type Setup = - static member Validation (validationCases: Test) = performTest validationCases + static member Metadata( + frontmatter: string + ) = + ValidationPackageMetadata.extractFromString frontmatter + + static member ValidationPackage( + metadata: ValidationPackageMetadata, + ?CriticalValidationCases: Test list, + ?NonCriticalValidationCases: Test list, + ?CQCHookEndpoint: string + ) = + ARCValidationPackage.create( + metadata = metadata, + ?CriticalValidationCasesList = CriticalValidationCases, + ?NonCriticalValidationCasesList = NonCriticalValidationCases, + ?CQCHookEndpoint = CQCHookEndpoint + ) + + static member ValidationPackage( + name: string, + summary: string, + description: string, + majorVersion: int, + minorVersion: int, + patchVersion: int, + ?Publish: bool, + ?Authors: Author array, + ?Tags: OntologyAnnotation array, + ?ReleaseNotes: string, + ?CriticalValidationCases: Test list, + ?NonCriticalValidationCases: Test list, + ?CQCHookEndpoint: string + ) = + Setup.ValidationPackage( + metadata = ValidationPackageMetadata.create( + name = name, + summary = summary, + description = description, + majorVersion = majorVersion, + minorVersion = minorVersion, + patchVersion = patchVersion, + ?Publish = Publish, + ?Authors = Authors, + ?Tags = Tags, + ?ReleaseNotes = ReleaseNotes + ), + ?CriticalValidationCases = CriticalValidationCases, + ?NonCriticalValidationCases = NonCriticalValidationCases, + ?CQCHookEndpoint = CQCHookEndpoint + ) - static member JUnitSummaryCreation( +type Execute = + +// ------------------ New API with ARCValidationPackage, metadata support and custom summaries ------------------ + static member Validation (arcValidationPackage: ARCValidationPackage) = + let criticalResults = performTest arcValidationPackage.CriticalValidationCases + let nonCriticalResults = performTest arcValidationPackage.NonCriticalValidationCases + + ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = criticalResults, + nonCriticalSummary = nonCriticalResults, + package = ValidationPackageSummary.create( + arcValidationPackage.Metadata, + ?HookEndpoint = arcValidationPackage.CQCHookEndpoint + ) + ) + + static member SummaryCreation( + path: string + ) = + fun (validationSummary: ValidationSummary) -> + ValidationSummary.writeJson path validationSummary + + static member JUnitReportCreation( path: string, ?Verbose: bool ) = let verbose = defaultArg Verbose false - fun (validationResults: Impl.TestRunSummary) -> writeJUnitSummary verbose path validationResults + + fun (validationSummary: ValidationSummary) -> + match validationSummary.Critical.OriginalRunSummary, validationSummary.NonCritical.OriginalRunSummary with + | None, None -> + printfn "No validation results to summarize" + | Some criticalResults, None -> + writeJUnitSummary verbose path criticalResults + | None, Some nonCriticalResults -> + writeJUnitSummary verbose path nonCriticalResults + | Some criticalResults, Some nonCriticalResults -> + combineTestRunSummaries [criticalResults; nonCriticalResults] + |> writeJUnitSummary verbose path static member BadgeCreation( path: string, @@ -22,9 +105,10 @@ type Execute = ?Thresholds: Map, ?DefaultColor: Color ) = - fun (validationResults: Impl.TestRunSummary) -> - validationResults - |> BadgeCreation.ofTestResults( + fun (validationSummary: ValidationSummary) -> + + validationSummary + |> BadgeCreation.ofValidationSummary( labelText, ?ValueSuffix = ValueSuffix, ?Thresholds = Thresholds, @@ -32,50 +116,114 @@ type Execute = ) |> fun b -> b.WriteBadge(path) - static member ValidationPipeline( - jUnitPath: string, - badgePath: string, - labelText: string, - ?ValueSuffix: string, - ?Thresholds: Map, - ?DefaultColor: Color - ) = - fun (validationCases: Test) -> - - let results = - validationCases - |> Execute.Validation - - results - |> Execute.JUnitSummaryCreation(jUnitPath) - - results - |> Execute.BadgeCreation(badgePath, labelText, ?ValueSuffix = ValueSuffix, ?Thresholds = Thresholds, ?DefaultColor = DefaultColor) - static member ValidationPipeline( basePath: string, - packageName: string, ?BadgeLabelText: string, ?ValueSuffix: string, ?Thresholds: Map, ?DefaultColor: Color ) = - fun (validationCases: Test) -> + fun (arcValidationPackage: ARCValidationPackage) -> + + let labelText = defaultArg BadgeLabelText arcValidationPackage.Metadata.Name + + let foldername = $"{arcValidationPackage.Metadata.Name}@{ValidationPackageMetadata.getSemanticVersionString arcValidationPackage.Metadata}" - let resultFolder = Path.Combine(basePath, ".arc-validate-results", packageName) + let resultFolder = Path.Combine(basePath, ".arc-validate-results", foldername) + let summaryPath = Path.Combine(resultFolder, "validation_summary.json") let badgePath = Path.Combine(resultFolder, "badge.svg") let jUnitPath = Path.Combine(resultFolder, "validation_report.xml") Directory.CreateDirectory(resultFolder) |> ignore let results = - validationCases + arcValidationPackage |> Execute.Validation + results |> Execute.SummaryCreation(summaryPath) + results |> Execute.JUnitReportCreation(jUnitPath) results - |> Execute.JUnitSummaryCreation(jUnitPath) + |> Execute.BadgeCreation( + badgePath, + labelText, + ?ValueSuffix = ValueSuffix, + ?Thresholds = Thresholds, + ?DefaultColor = DefaultColor + ) - let labelText = defaultArg BadgeLabelText packageName +// ------------------ Legacy API without ARCValidationPackage, metadata, or custom Summaries ------------------ + + //static member Validation (validationCases: Test) = performTest validationCases + + //static member JUnitSummaryCreation( + // path: string, + // ?Verbose: bool + //) = + // let verbose = defaultArg Verbose false + // fun (validationResults: Impl.TestRunSummary) -> writeJUnitSummary verbose path validationResults + + //static member BadgeCreation( + // path: string, + // labelText: string, + // ?ValueSuffix: string, + // ?Thresholds: Map, + // ?DefaultColor: Color + //) = + // fun (validationResults: Impl.TestRunSummary) -> + // validationResults + // |> BadgeCreation.ofTestResults( + // labelText, + // ?ValueSuffix = ValueSuffix, + // ?Thresholds = Thresholds, + // ?DefaultColor = DefaultColor + // ) + // |> fun b -> b.WriteBadge(path) + + //static member ValidationPipeline( + // jUnitPath: string, + // badgePath: string, + // labelText: string, + // ?ValueSuffix: string, + // ?Thresholds: Map, + // ?DefaultColor: Color + //) = + // fun (validationCases: Test) -> + + // let results = + // validationCases + // |> Execute.Validation + + // results + // |> Execute.JUnitSummaryCreation(jUnitPath) + + // results + // |> Execute.BadgeCreation(badgePath, labelText, ?ValueSuffix = ValueSuffix, ?Thresholds = Thresholds, ?DefaultColor = DefaultColor) + + //static member ValidationPipeline( + // basePath: string, + // packageName: string, + // ?BadgeLabelText: string, + // ?ValueSuffix: string, + // ?Thresholds: Map, + // ?DefaultColor: Color + //) = + // fun (validationCases: Test) -> + + // let resultFolder = Path.Combine(basePath, ".arc-validate-results", packageName) + // let badgePath = Path.Combine(resultFolder, "badge.svg") + // let jUnitPath = Path.Combine(resultFolder, "validation_report.xml") + + // Directory.CreateDirectory(resultFolder) |> ignore + + // let results = + // validationCases + // |> Execute.Validation + + // results + // |> Execute.JUnitSummaryCreation(jUnitPath) + + // let labelText = defaultArg BadgeLabelText packageName + + // results + // |> Execute.BadgeCreation(badgePath, labelText, ?ValueSuffix = ValueSuffix, ?Thresholds = Thresholds, ?DefaultColor = DefaultColor) - results - |> Execute.BadgeCreation(badgePath, labelText, ?ValueSuffix = ValueSuffix, ?Thresholds = Thresholds, ?DefaultColor = DefaultColor) \ No newline at end of file diff --git a/src/ARCExpect/ValidationSummary.fs b/src/ARCExpect/ValidationSummary.fs new file mode 100644 index 0000000..6508581 --- /dev/null +++ b/src/ARCExpect/ValidationSummary.fs @@ -0,0 +1,129 @@ +namespace ARCExpect + +open Expecto +open System.Text.Json +open System.Text.Json.Serialization +open System.IO +open AVPRIndex + +/// +/// Represents a brief summary of the result of validating an ARC against a set of validation cases. +/// +type ValidationResult = { + HasFailures: bool + Total: int + Passed: int + Failed: int + Errored: int + [] + OriginalRunSummary: Impl.TestRunSummary option +} with + static member create( + hasFailures: bool, + total: int, + passed: int, + failed: int, + errored: int, + ?OriginalRunSummary: Impl.TestRunSummary + ) = { + HasFailures = hasFailures + Total = total + Passed = passed + Failed = failed + Errored = errored + OriginalRunSummary = OriginalRunSummary + } + + static member create (total: int, passed: int, failed: int, errored: int, ?OriginalRunSummary: Impl.TestRunSummary) = + ValidationResult.create( + hasFailures = (failed > 0 || errored > 0), + total = total, + passed = passed, + failed = failed, + errored = errored, + ?OriginalRunSummary = OriginalRunSummary + ) + + static member ofExpectoTestRunSummary (summary: Impl.TestRunSummary) = + + let totalTests = summary.errored @ summary.failed @ summary.ignored @ summary.passed + + ValidationResult.create( + total = totalTests.Length, + passed = summary.passed.Length, + errored = summary.errored.Length, + failed = summary.failed.Length, + OriginalRunSummary = summary + ) + +/// +/// Represents a brief summary of a validation package. Should be expanded to include full package metadata in the future. +/// +type ValidationPackageSummary = { + Name: string + Version: string + Summary: string + Description: string + HookEndpoint: string option +} with + static member create( + name: string, + version: string, + summary: string, + description: string, + ?HookEndpoint: string + ) = { + Name = name + Version = version + Summary = summary + Description = description + HookEndpoint = HookEndpoint + } + static member create ( + metadata: ValidationPackageMetadata, + ?HookEndpoint: string + ) = + ValidationPackageSummary.create( + name = metadata.Name, + version = ValidationPackageMetadata.getSemanticVersionString metadata, + summary = metadata.Summary, + description = metadata.Description, + ?HookEndpoint = HookEndpoint + ) + +/// +/// Represents a summary of the validation results of an ARC against a validation package containing critical and non-critical validation cases. +/// +type ValidationSummary = { + Critical: ValidationResult + NonCritical: ValidationResult + ValidationPackage: ValidationPackageSummary +} with + static member create( + critical: ValidationResult, + nonCritical: ValidationResult, + validationPackage: ValidationPackageSummary + ) = { + Critical = critical + NonCritical = nonCritical + ValidationPackage = validationPackage + } + static member ofExpectoTestRunSummaries ( + criticalSummary: Impl.TestRunSummary, + nonCriticalSummary: Impl.TestRunSummary, + package: ValidationPackageSummary + ) = + ValidationSummary.create( + critical = ValidationResult.ofExpectoTestRunSummary criticalSummary, + nonCritical = ValidationResult.ofExpectoTestRunSummary nonCriticalSummary, + validationPackage = package + ) + + static member toJson (summary: ValidationSummary) = + JsonSerializer.Serialize(summary, JsonOptions.options) + + static member fromJson (json: string) = + JsonSerializer.Deserialize(json, JsonOptions.options) + + static member writeJson (path: string) (summary: ValidationSummary) = + File.WriteAllText(path, ValidationSummary.toJson summary) \ No newline at end of file diff --git a/src/arc-validate/APIs/ValidateAPI.fs b/src/arc-validate/APIs/ValidateAPI.fs index e102d60..3acbc3d 100644 --- a/src/arc-validate/APIs/ValidateAPI.fs +++ b/src/arc-validate/APIs/ValidateAPI.fs @@ -3,7 +3,6 @@ open ARCValidate open ARCValidate.CLIArguments open ARCExpect -open ARCExpect.Configs open ARCTokenization open ARCValidationPackages @@ -51,38 +50,30 @@ module ValidateAPI = ] |> List.iter AnsiConsole.MarkupLine + let specVersion = defaultArg (args.TryGetResult(Specification_Version)) "latest" let status = AnsiConsole.Status() let mutable exitCode = ExitCode.Success - status.Start($"Performing validation against the ARC specification V2_Draft", fun ctx -> + status.Start($"Performing validation against version '{specVersion}' of the ARC specification", fun ctx -> + + let version_used, validationCases = SpecificationValidation.SpecificationSelection.tryGetValidationCasesForSpecificationVersion specVersion root + + if specVersion = "latest" then printfn $"latest spec version supported is {version_used}" if verbose then AnsiConsole.MarkupLine("LOG: Running in:") AnsiConsole.Write(TextPath(Path.GetFullPath(root))) AnsiConsole.MarkupLine("") - AnsiConsole.MarkupLine($"LOG: running validation against [bold underline green]ARC specification V2_Draft[/].") + AnsiConsole.MarkupLine($"LOG: running validation against [bold underline green] version '{specVersion}' of the ARC specification[/].") AnsiConsole.MarkupLine($"LOG: Output path is:") AnsiConsole.Write(TextPath(Path.GetFullPath(outPath))) AnsiConsole.MarkupLine("") - let cases = ARCSpecification.V2_Draft.validationCases root - - let outDirBadge = System.IO.Path.Combine(root, "ARC_specification_V2_Draft.svg") - let outDirResXml =System.IO.Path.Combine(root, "ARC_specification_V2_Draft.xml") - - let results = ARCExpect.Execute.Validation cases - - results - |> Execute.JUnitSummaryCreation(outDirResXml) - - results - |> Execute.BadgeCreation(outDirBadge, "ARC specification V2_Draft", "passing", DefaultColor=AnyBadge.NET.Color.GREEN) + validationCases + |> Execute.ValidationPipeline outPath - if results.successful then - exitCode <- ExitCode.Success - else - exitCode <- ExitCode.CriticalTestFailure + exitCode <- ExitCode.Success ) | Some packageName -> // Validate against a specific package diff --git a/src/arc-validate/CLIArgs/ValidateArgs.fs b/src/arc-validate/CLIArgs/ValidateArgs.fs index 1cd08bf..230a4e9 100644 --- a/src/arc-validate/CLIArgs/ValidateArgs.fs +++ b/src/arc-validate/CLIArgs/ValidateArgs.fs @@ -6,6 +6,7 @@ type ValidateArgs = | [] Out_Directory of path:string | [] Package of package_name:string | [] Package_Version of package_version:string + | [] Specification_Version of specification_version: string | [] Preview interface IArgParserTemplate with @@ -15,4 +16,5 @@ type ValidateArgs = | ARC_Directory _ -> "Optional. Specify a directory that contains the arc to convert. Default: content of the ARC_PATH environment variable. If ARC_PATH is not set: current directory." | Package _ -> "Optional. Specify a validation package to use on top of the default validation for invenio export. Default: no package is used, only structural validation for invenio export." | Package_Version _ -> "Optional. Specify a version of the validation package to use. If no version is specified, the latest version will be used." + | Specification_Version _ -> "Optional. Only has an effect if no package is specified via '-p' Specify a version of the ARC specification to validate against. Default: 'latest'." | Preview -> "Optional. Use the preview version of the package." \ No newline at end of file diff --git a/src/arc-validate/Program.fs b/src/arc-validate/Program.fs index 1360fbc..1a0fa2d 100644 --- a/src/arc-validate/Program.fs +++ b/src/arc-validate/Program.fs @@ -4,7 +4,6 @@ open Expecto open Argu open System.IO open ARCExpect -open ARCExpect.Configs open ARCValidate open ARCValidate.CLICommands diff --git a/tests/ARCExpect.Tests/ARCExpect.Tests.fsproj b/tests/ARCExpect.Tests/ARCExpect.Tests.fsproj index b5fd169..e659a08 100644 --- a/tests/ARCExpect.Tests/ARCExpect.Tests.fsproj +++ b/tests/ARCExpect.Tests/ARCExpect.Tests.fsproj @@ -1,25 +1,29 @@ - + + Exe net8.0 false + + - - + + + @@ -28,4 +32,4 @@ - \ No newline at end of file + diff --git a/tests/ARCExpect.Tests/ExpectoExtensionsTests.fs b/tests/ARCExpect.Tests/ExpectoExtensionsTests.fs index 1e9f3e4..5bf381f 100644 --- a/tests/ARCExpect.Tests/ExpectoExtensionsTests.fs +++ b/tests/ARCExpect.Tests/ExpectoExtensionsTests.fs @@ -1,7 +1,6 @@ module ExpectoExtensionsTests open ARCExpect -open ARCExpect.Configs open Expecto diff --git a/tests/ARCExpect.Tests/PathConfigTests.fs b/tests/ARCExpect.Tests/PathConfigTests.fs deleted file mode 100644 index 4572bdc..0000000 --- a/tests/ARCExpect.Tests/PathConfigTests.fs +++ /dev/null @@ -1,54 +0,0 @@ -module PathConfigTests - -open ARCExpect -open ARCExpect.Configs - -open Expecto - -let pathFixture rootPath = - fun f -> - let pathConfig = PathConfig(rootPath) - f pathConfig - -let normalizePath (path: string) = path.Replace("\\","/") - -[] -let ``PathConfig tests`` = - testList "PathConfig" [ - yield! testFixture (pathFixture "/arcs/test/path") [ - "PathConfig.ArcRootPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ARCRootPath ) "/arcs/test/path" "ArcRootPath was incorrect") - "PathConfig.StudiesPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.StudiesPath ) "/arcs/test/path/studies" "StudiesPath was incorrect") - "PathConfig.AssaysPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.AssaysPath ) "/arcs/test/path/assays" "AssaysPath was incorrect") - "PathConfig.RunsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.RunsPath ) "/arcs/test/path/runs" "RunsPath was incorrect") - "PathConfig.WorkflowsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.WorkflowsPath ) "/arcs/test/path/workflows" "WorkflowsPath was incorrect") - "PathConfig.InvestigationPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.InvestigationPath ) "/arcs/test/path/isa.investigation.xlsx" "InvestigationPath was incorrect") - "PathConfig.DotArcFolderPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.DotARCFolderPath ) "/arcs/test/path/.arc" "DotArcFolderPath was incorrect") - "PathConfig.GitFolderPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.GitFolderPath ) "/arcs/test/path/.git" "GitFolderPath was incorrect") - "PathConfig.ConfigPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ConfigPath ) "/arcs/test/path/.git/config" "ConfigPath was incorrect") - "PathConfig.DescriptionPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.DescriptionPath ) "/arcs/test/path/.git/description" "DescriptionPath was incorrect") - "PathConfig.HeadPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.HeadPath ) "/arcs/test/path/.git/HEAD" "HeadPath was incorrect") - "PathConfig.ObjectsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ObjectsPath ) "/arcs/test/path/.git/objects" "ObjectsPath was incorrect") - "PathConfig.ObjectsInfoPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ObjectsInfoPath ) "/arcs/test/path/.git/objects/info" "ObjectsInfoPath was incorrect") - "PathConfig.ObjectsPackPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ObjectsPackPath ) "/arcs/test/path/.git/objects/pack" "ObjectsPackPath was incorrect") - "PathConfig.InfoPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.InfoPath ) "/arcs/test/path/.git/info" "InfoPath was incorrect") - "PathConfig.ExcludePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ExcludePath ) "/arcs/test/path/.git/info/exclude" "ExcludePath was incorrect") - "PathConfig.RefsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.RefsPath ) "/arcs/test/path/.git/refs" "RefsPath was incorrect") - "PathConfig.RefsHeadsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.RefsHeadsPath ) "/arcs/test/path/.git/refs/heads" "RefsHeadsPath was incorrect") - "PathConfig.RefsTagsPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.RefsTagsPath ) "/arcs/test/path/.git/refs/tags" "RefsTagsPath was incorrect") - "PathConfig.HooksPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.HooksPath ) "/arcs/test/path/.git/hooks" "HooksPath was incorrect") - "PathConfig.ApplyPatchPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.ApplyPatchPath ) "/arcs/test/path/.git/hooks/applypatch-msg.sample" "ApplyPatchPath was incorrect") - "PathConfig.CommitSamplePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.CommitSamplePath ) "/arcs/test/path/.git/hooks/commit-msg.sample" "CommitSamplePath was incorrect") - "PathConfig.FsmonitorPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.FsmonitorPath ) "/arcs/test/path/.git/hooks/fsmonitor-watchman.sample" "FsmonitorPath was incorrect") - "PathConfig.PostUpdatePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PostUpdatePath ) "/arcs/test/path/.git/hooks/post-update.sample" "PostUpdatePath was incorrect") - "PathConfig.PreApplyPatchPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PreApplyPatchPath ) "/arcs/test/path/.git/hooks/pre-applypatch.sample" "PreApplyPatchPath was incorrect") - "PathConfig.PreCommitPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PreCommitPath ) "/arcs/test/path/.git/hooks/pre-commit.sample" "PreCommitPath was incorrect") - "PathConfig.PreMergeCommitPath", (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PreMergeCommitPath) "/arcs/test/path/.git/hooks/pre-merge-commit.sample" "PreMergeCommitPath was incorrect") - "PathConfig.PrePushPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PrePushPath ) "/arcs/test/path/.git/hooks/pre-push.sample" "PrePushPath was incorrect") - "PathConfig.PreRebasePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PreRebasePath ) "/arcs/test/path/.git/hooks/pre-rebase.sample" "PreRebasePath was incorrect") - "PathConfig.PreReceivePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PreReceivePath ) "/arcs/test/path/.git/hooks/pre-receive.sample" "PreReceivePath was incorrect") - "PathConfig.PrepareCommitPath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PrepareCommitPath ) "/arcs/test/path/.git/hooks/prepare-commit-msg.sample" "PrepareCommitPath was incorrect") - "PathConfig.PushToCheckoutPath", (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.PushToCheckoutPath) "/arcs/test/path/.git/hooks/push-to-checkout.sample" "PushToCheckoutPath was incorrect") - "PathConfig.UpdatePath" , (fun pathConfig -> fun () -> Expect.equal (normalizePath pathConfig.UpdatePath ) "/arcs/test/path/.git/hooks/update.sample" "UpdatePath was incorrect") - ] - ] - \ No newline at end of file diff --git a/tests/ARCExpect.Tests/ReferenceObjects.fs b/tests/ARCExpect.Tests/ReferenceObjects.fs index 567a8a5..f876f88 100644 --- a/tests/ARCExpect.Tests/ReferenceObjects.fs +++ b/tests/ARCExpect.Tests/ReferenceObjects.fs @@ -1,6 +1,15 @@ module ReferenceObjects open ControlledVocabulary +open ARCExpect +open AVPRIndex + +open Expecto + +module TestCase = + + let dummyTestWillPass = testCase "dummyTest1" (fun _ -> Expect.isTrue true "is not true") + let dummyTestWillFail = testCase "dummyTest2" (fun _ -> Expect.isTrue false "is not true") module CvTerms = @@ -16,4 +25,220 @@ module CvParams = let ``Investigation Person Email (valid)`` = CvParam(CvTerms.``Investigation Person Email``, ParamValue.Value "yes@yes.com") - let ``Investigation Person Email (invalid)`` = CvParam(CvTerms.``Investigation Person Email``, ParamValue.Value "nope") \ No newline at end of file + let ``Investigation Person Email (invalid)`` = CvParam(CvTerms.``Investigation Person Email``, ParamValue.Value "nope") + +module ValidationResult = + + let allPassed = { + HasFailures = false + Total = 1 + Passed = 1 + Failed = 0 + Errored = 0 + OriginalRunSummary = None + } + + let allFailed = { + HasFailures = true + Total = 1 + Passed = 0 + Failed = 1 + Errored = 0 + OriginalRunSummary = None + } + +module ValidationPackageSummary = + + let noHook = { + Name = "test" + Version = "1.0.0" + Summary = "A package without CQC hook." + Description = "A package without CQC hook. More text here." + HookEndpoint = None + } + + let withHook = { + Name = "test" + Version = "1.0.0" + Summary = "A package with CQC hook." + Description = "A package with CQC hook. More text here." + HookEndpoint = Some "http://test.com" + } + +module ValidationSummary = + + let allPassedNoHook = + { + Critical = ValidationResult.allPassed + NonCritical = ValidationResult.allPassed + ValidationPackage = ValidationPackageSummary.noHook + } + + let allPassedNoHookJson = """{"Critical":{"HasFailures":false,"Total":1,"Passed":1,"Failed":0,"Errored":0},"NonCritical":{"HasFailures":false,"Total":1,"Passed":1,"Failed":0,"Errored":0},"ValidationPackage":{"Name":"test","Version":"1.0.0","Summary":"A package without CQC hook.","Description":"A package without CQC hook. More text here."}}""" + + let allPassedWithHook = + { + Critical = ValidationResult.allPassed + NonCritical = ValidationResult.allPassed + ValidationPackage = ValidationPackageSummary.withHook + } + + let allPassedWithHookJson = """{"Critical":{"HasFailures":false,"Total":1,"Passed":1,"Failed":0,"Errored":0},"NonCritical":{"HasFailures":false,"Total":1,"Passed":1,"Failed":0,"Errored":0},"ValidationPackage":{"Name":"test","Version":"1.0.0","Summary":"A package with CQC hook.","Description":"A package with CQC hook. More text here.","HookEndpoint":"http://test.com"}}""" + + let allFailedNoHook = + { + Critical = ValidationResult.allFailed + NonCritical = ValidationResult.allFailed + ValidationPackage = ValidationPackageSummary.noHook + } + + let allFailedWithHook = + { + Critical = ValidationResult.allFailed + NonCritical = ValidationResult.allFailed + ValidationPackage = ValidationPackageSummary.withHook + } + + let nonCriticalFailedNoHook = + { + Critical = ValidationResult.allPassed + NonCritical = ValidationResult.allFailed + ValidationPackage = ValidationPackageSummary.noHook + } + + let nonCriticalFailedWithHook = + { + Critical = ValidationResult.allPassed + NonCritical = ValidationResult.allFailed + ValidationPackage = ValidationPackageSummary.withHook + } + + let criticalFailedNoHook = + { + Critical = ValidationResult.allFailed + NonCritical = ValidationResult.allPassed + ValidationPackage = ValidationPackageSummary.noHook + } + + let criticalFailedWithHook = + { + Critical = ValidationResult.allFailed + NonCritical = ValidationResult.allPassed + ValidationPackage = ValidationPackageSummary.withHook + } + +module Frontmatter = + + let validNoHook = """(* +--- +Name: test +MajorVersion: 1 +MinorVersion: 0 +PatchVersion: 0 +Summary: A package without CQC hook. +Description: A package without CQC hook. More text here. +--- +*)""" .ReplaceLineEndings("\n") + + let validWithHook = """(* +--- +Name: test +MajorVersion: 1 +MinorVersion: 0 +PatchVersion: 0 +Summary: A package with CQC hook. +Description: A package with CQC hook. More text here. +--- +*)""" .ReplaceLineEndings("\n") + + let invalid = """ +Name: invalid +MinorVersion: 0 +PatchVersion: 0 +Summary: My package does the thing. +Description: | + My package does the thing. + It does it very good, it does it very well. + It does it very fast, it does it very swell. + +""" .ReplaceLineEndings("\n") + +module ValidationPackageMetadata = + + let validNoHook = + + ValidationPackageMetadata( + Name = "test", + MajorVersion = 1, + MinorVersion = 0, + PatchVersion = 0, + Summary = "A package without CQC hook.", + Description = "A package without CQC hook. More text here." + ) + + let validWithHook = + + ValidationPackageMetadata( + Name = "test", + MajorVersion = 1, + MinorVersion = 0, + PatchVersion = 0, + Summary = "A package with CQC hook.", + Description = "A package with CQC hook. More text here." + ) + +module JUnitReport = + + let allPassedWithHookXml = """""".ReplaceLineEndings("\n") + +module Badge = + + let allPassedWithHookBadgeSVG = """ + + + + + + + + + + + + + + + allPassedWithHook + allPassedWithHook + + + 2/2 + 2/2 + + +""" .ReplaceLineEndings("\n") + + let allPassedWithHookBadgeSVGFromPipeline = """ + + + + + + + + + + + + + + + test + test + + + 2/2 + 2/2 + + +""" .ReplaceLineEndings("\n") diff --git a/tests/ARCExpect.Tests/TestUtils.fs b/tests/ARCExpect.Tests/TestUtils.fs new file mode 100644 index 0000000..d6e6858 --- /dev/null +++ b/tests/ARCExpect.Tests/TestUtils.fs @@ -0,0 +1,25 @@ +module TestUtils + +open Expecto +open ARCExpect +open AVPRIndex + +module Expect = + + let validationResultEqualIgnoringOriginal (actual: ARCExpect.ValidationResult) (expected: ARCExpect.ValidationResult) = + let actual = { actual with OriginalRunSummary = None } + let expected = { expected with OriginalRunSummary = None } + Expect.isTrue (actual = expected) "validation results were not equal ignoring the original run summaries." + + let validationSummaryEqualIgnoringOriginal (actual: ARCExpect.ValidationSummary) (expected: ARCExpect.ValidationSummary) = + let actual = { + actual with + Critical.OriginalRunSummary = None + NonCritical.OriginalRunSummary = None + } + let expected = { + expected with + Critical.OriginalRunSummary = None + NonCritical.OriginalRunSummary = None + } + Expect.equal actual expected "validation summaries were not equal ignoring the original run summaries." \ No newline at end of file diff --git a/tests/ARCExpect.Tests/TopLevelAPITests.fs b/tests/ARCExpect.Tests/TopLevelAPITests.fs new file mode 100644 index 0000000..539abf9 --- /dev/null +++ b/tests/ARCExpect.Tests/TopLevelAPITests.fs @@ -0,0 +1,154 @@ +module TopLevelAPITests + +open ARCExpect +open AVPRIndex +open Expecto +open TestUtils +open System.IO + +[] +let ``Toplevel API Setup tests`` = + testSequenced (testList "Toplevel API tests" [ + testList "Setup_Metadata" [ + test "correct metadata is extracted from valid frontmatter string" { + let actual = Setup.Metadata(ReferenceObjects.Frontmatter.validNoHook) + Expect.equal actual ReferenceObjects.ValidationPackageMetadata.validNoHook "metadata was not equal" + } + test "incorrect frontmatter string throws" { + Expect.throws (fun () -> Setup.Metadata(ReferenceObjects.Frontmatter.invalid) |> ignore) "did not throw" + } + ] + testList "Setup_ValidationPackage" [ + test "validation package created from metadata and equivalent single values are equal" { + let fromMetadata = + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validNoHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillFail] + ) + let fromValues = + Setup.ValidationPackage( + name = "test", + majorVersion = 1, + minorVersion = 0, + patchVersion = 0, + summary = "A package without CQC hook.", + description = "A package without CQC hook. More text here.", + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillFail] + ) + Expect.equal fromMetadata.Metadata.Name fromValues.Metadata.Name "metadata names were not equal" + Expect.equal fromMetadata.Metadata.MajorVersion fromValues.Metadata.MajorVersion "metadata major versions were not equal" + Expect.equal fromMetadata.Metadata.MinorVersion fromValues.Metadata.MinorVersion "metadata minor versions were not equal" + Expect.equal fromMetadata.Metadata.PatchVersion fromValues.Metadata.PatchVersion "metadata patch versions were not equal" + Expect.equal fromMetadata.Metadata.Summary fromValues.Metadata.Summary "metadata summaries were not equal" + Expect.equal fromMetadata.Metadata.Description fromValues.Metadata.Description "metadata descriptions were not equal" + Expect.equal fromMetadata.Metadata.Authors fromValues.Metadata.Authors "metadata authors were not equal" + Expect.equal fromMetadata.Metadata.Publish fromValues.Metadata.Publish "metadata publish were not equal" + Expect.equal fromMetadata.Metadata.Tags fromValues.Metadata.Tags "metadata tags were not equal" + Expect.equal fromMetadata.Metadata.ReleaseNotes fromValues.Metadata.ReleaseNotes "metadata release notes were not equal" + + Expect.equal fromMetadata.CQCHookEndpoint fromValues.CQCHookEndpoint "hook endpoint was not equal" + } + ] + testList "Execute_Validation" [ + test "resulting summary is correct noHook" { + let actual = + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validNoHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass] + ) + |> Execute.Validation + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allPassedNoHook + } + + test "resulting summary is correct with hook" { + let actual = + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validWithHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + CQCHookEndpoint = "http://test.com" + ) + |> Execute.Validation + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allPassedWithHook + } + ] + testList "Execute_SummaryCreation" [ + test "Correct summary file is created" { + let path = Path.GetTempFileName() |> fun p -> Path.ChangeExtension(p, "json") + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validWithHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + CQCHookEndpoint = "http://test.com" + ) + |> Execute.Validation + |> Execute.SummaryCreation path + + let actual = + File.ReadAllText path + + Expect.equal actual ReferenceObjects.ValidationSummary.allPassedWithHookJson "summary files were not equal" + } + ] + testList "Execute_JUnitReportCreation" [ + test "Correct report file is created" { + let path = Path.GetTempFileName() |> fun p -> Path.ChangeExtension(p, "xml") + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validWithHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + CQCHookEndpoint = "http://test.com" + ) + |> Execute.Validation + |> Execute.JUnitReportCreation path + + let actual = (File.ReadAllText path).ReplaceLineEndings("\n") + + Expect.equal actual ReferenceObjects.JUnitReport.allPassedWithHookXml "report files were not equal" + + } + ] + testList "Execute_BadgeCreation" [ + test "Correct badge file is created" { + let path = Path.GetTempFileName() |> fun p -> Path.ChangeExtension(p, "svg") + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validWithHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + CQCHookEndpoint = "http://test.com" + ) + |> Execute.Validation + |> Execute.BadgeCreation(path, labelText="allPassedWithHook") + + let actual = (File.ReadAllText path).ReplaceLineEndings("\n") + + Expect.equal actual ReferenceObjects.Badge.allPassedWithHookBadgeSVG "badge files were not equal" + } + ] + testList "Execute_ValidationPipeline" [ + test "Correct output folder is created" { + let path = Path.GetTempPath() + Setup.ValidationPackage( + metadata = ReferenceObjects.ValidationPackageMetadata.validWithHook, + CriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + NonCriticalValidationCases = [ReferenceObjects.TestCase.dummyTestWillPass], + CQCHookEndpoint = "http://test.com" + ) + |> Execute.ValidationPipeline( + basePath = path + ) + let expectedFolderName = Path.Combine(path, ".arc-validate-results/test@1.0.0") + let actualSummary = File.ReadAllText (Path.Combine(expectedFolderName, "validation_summary.json")) + let actualReport = (File.ReadAllText (Path.Combine(expectedFolderName, "validation_report.xml"))).ReplaceLineEndings("\n") + let actualBadge = (File.ReadAllText (Path.Combine(expectedFolderName, "badge.svg"))).ReplaceLineEndings("\n") + + Expect.isTrue (Path.Exists(expectedFolderName)) "Output path did not exist (.arc-validate-results/@)" + Expect.equal actualSummary ReferenceObjects.ValidationSummary.allPassedWithHookJson "summary files were not equal" + Expect.equal actualReport ReferenceObjects.JUnitReport.allPassedWithHookXml "report files were not equal" + Expect.equal actualBadge ReferenceObjects.Badge.allPassedWithHookBadgeSVGFromPipeline "badge files were not equal" + } + ] + ]) \ No newline at end of file diff --git a/tests/ARCExpect.Tests/ValidationSummaryTests.fs b/tests/ARCExpect.Tests/ValidationSummaryTests.fs new file mode 100644 index 0000000..5c3f706 --- /dev/null +++ b/tests/ARCExpect.Tests/ValidationSummaryTests.fs @@ -0,0 +1,182 @@ +module ValidationSummaryTests + +open ARCExpect +open Expecto +open TestUtils +open AVPRIndex + + +let dummyTestPassed = ReferenceObjects.TestCase.dummyTestWillPass |> performTest +let dummyTestFailed = ReferenceObjects.TestCase.dummyTestWillFail |> performTest + +let testPackageName = "test" +let testPackageVersion = "1.0.0" + +let testSummaryNoHook = "A package without CQC hook." +let testDescriptionNoHook = "A package without CQC hook. More text here." + +let testSummaryWithHook = "A package with CQC hook." +let testDescriptionWithHook = "A package with CQC hook. More text here." +let testHook = "http://test.com" + +let testPackageNoHook = ValidationPackageSummary.create( + name = testPackageName, + version = testPackageVersion, + summary = testSummaryNoHook, + description = testDescriptionNoHook +) + +let testPackageWithHook = ValidationPackageSummary.create( + name = testPackageName, + version = testPackageVersion, + summary = testSummaryWithHook, + description = testDescriptionWithHook, + HookEndpoint = testHook +) + +[] +let ``ValidationResult tests`` = + testList "ValidationSummary tests" [ + testList "ValidationResult" [ + test "correct result is created from passed TestRunSummary" { + let actual = ValidationResult.ofExpectoTestRunSummary dummyTestPassed + Expect.validationResultEqualIgnoringOriginal actual ReferenceObjects.ValidationResult.allPassed + } + test "correct result is created from failed TestRunSummary" { + let actual = ValidationResult.ofExpectoTestRunSummary dummyTestFailed + Expect.validationResultEqualIgnoringOriginal actual ReferenceObjects.ValidationResult.allFailed + } + ] + testList "ValidationPackageSummary" [ + test "package summary is created correctly without hook" { + let actual = ValidationPackageSummary.create( + name = testPackageName, + version = testPackageVersion, + summary = testSummaryNoHook, + description = testDescriptionNoHook + ) + Expect.equal actual ReferenceObjects.ValidationPackageSummary.noHook "package summary was not equal" + } + test "package summary is created correctly with hook" { + let actual = ValidationPackageSummary.create( + name = testPackageName, + version = testPackageVersion, + summary = testSummaryWithHook, + description = testDescriptionWithHook, + HookEndpoint = testHook + ) + Expect.equal actual ReferenceObjects.ValidationPackageSummary.withHook "package summary was not equal" + } + ] + testList "ValidationSummary" [ + test "critial passed, noncritical passed, package with no hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestPassed, + nonCriticalSummary = dummyTestPassed, + package = testPackageNoHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allPassedNoHook + } + test "critial passed, noncritical passed, package with hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestPassed, + nonCriticalSummary = dummyTestPassed, + package = testPackageWithHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allPassedWithHook + } + test "critial passed, noncritical failed, package with no hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestPassed, + nonCriticalSummary = dummyTestFailed, + package = testPackageNoHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.nonCriticalFailedNoHook + } + test "critial passed, noncritical failed, package with hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestPassed, + nonCriticalSummary = dummyTestFailed, + package = testPackageWithHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.nonCriticalFailedWithHook + } + + test "critial failed, noncritical passed, package with no hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestFailed, + nonCriticalSummary = dummyTestPassed, + package = testPackageNoHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.criticalFailedNoHook + } + test "critial failed, noncritical passed, package with hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestFailed, + nonCriticalSummary = dummyTestPassed, + package = testPackageWithHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.criticalFailedWithHook + } + test "critial failed, noncritical failed, package with no hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestFailed, + nonCriticalSummary = dummyTestFailed, + package = testPackageNoHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allFailedNoHook + } + test "critial failed, noncritical failed, package with hook is created correctly from TestRunSummaries" { + let actual = ValidationSummary.ofExpectoTestRunSummaries( + criticalSummary = dummyTestFailed, + nonCriticalSummary = dummyTestFailed, + package = testPackageWithHook + ) + Expect.validationSummaryEqualIgnoringOriginal actual ReferenceObjects.ValidationSummary.allFailedWithHook + } + ] + testList "Serialization" [ + test "correctly serialized without hook" { + let actual = + ReferenceObjects.ValidationSummary.allPassedNoHook + |> ValidationSummary.toJson + Expect.equal actual ReferenceObjects.ValidationSummary.allPassedNoHookJson "serialization was not equal" + } + test "roundtrip without hook" { + let actual = + ReferenceObjects.ValidationSummary.allPassedNoHook + |> ValidationSummary.toJson + |> ValidationSummary.fromJson + + Expect.equal actual ReferenceObjects.ValidationSummary.allPassedNoHook "roundtrip was not equal" + } + test "correctly serialized with hook" { + let actual = + ReferenceObjects.ValidationSummary.allPassedWithHook + |> ValidationSummary.toJson + Expect.equal actual ReferenceObjects.ValidationSummary.allPassedWithHookJson "serialization was not equal" + } + test "roundtrip with hook" { + let actual = + ReferenceObjects.ValidationSummary.allPassedWithHook + |> ValidationSummary.toJson + |> ValidationSummary.fromJson + + Expect.equal actual ReferenceObjects.ValidationSummary.allPassedWithHook "roundtrip was not equal" + } + test "roundtrip looses original summary" { + + let initial = ValidationSummary.ofExpectoTestRunSummaries(dummyTestPassed, dummyTestPassed, ReferenceObjects.ValidationPackageSummary.noHook) + let actual = + ValidationSummary.ofExpectoTestRunSummaries(dummyTestPassed, dummyTestPassed, ReferenceObjects.ValidationPackageSummary.noHook) + |> ValidationSummary.toJson + |> ValidationSummary.fromJson + + Expect.isSome initial.Critical.OriginalRunSummary "OriginalRunSummary was not some initially" + Expect.isNone actual.Critical.OriginalRunSummary "OriginalRunSummary was not none after json roundtrip" + Expect.isSome initial.NonCritical.OriginalRunSummary "OriginalRunSummary was not some initially" + Expect.isNone actual.NonCritical.OriginalRunSummary "OriginalRunSummary was not none after json roundtrip" + + } + ] + ] \ No newline at end of file diff --git a/tests/arc-validate.Tests/CLITests/ValidateCommandTests.fs b/tests/arc-validate.Tests/CLITests/ValidateCommandTests.fs index 8a145b6..bfbc9c1 100644 --- a/tests/arc-validate.Tests/CLITests/ValidateCommandTests.fs +++ b/tests/arc-validate.Tests/CLITests/ValidateCommandTests.fs @@ -8,6 +8,8 @@ open System.IO open System.Diagnostics open Fake.Core +open ARCExpect + open Common open Common.TestUtils open TestUtils @@ -17,7 +19,7 @@ open JUnit [] let ``ValidateCommand CLI Tests`` = testSequenced (testList "arc-validate validate" [ - testSequenced (testList "preview" [ + testSequenced (testList "preview ARCExpect < 2" [ testSequenced (testList "package test version 2" [ // run: // - arc-validate --verbose package install test -v 2.0.0 --preview @@ -89,7 +91,7 @@ let ``ValidateCommand CLI Tests`` = ] ]) ]) - testSequenced (testList "avpr" [ + testSequenced (testList "avpr ARCExpect < 2" [ testSequenced (testList "package test version 2" [ yield! // run: @@ -157,49 +159,90 @@ let ``ValidateCommand CLI Tests`` = fun tool args proc -> Expect.isFalse (proc.Result.Output.Contains("Package test not installed. You can run run arc-validate package install ")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) "Console Output is correct" , fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("If you can read this in your console, you successfully executed test package v3.0.0!")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) - ] ]) - ]) - // printfn "%s" (Path.GetFullPath("fixtures/arcs/inveniotestarc")) - - // let proc = runTool "../../../../../publish/arc-validate" [|"validate"; "-i"; "fixtures/arcs/inveniotestarc"|] - - // printfn "%s" proc.Result.Output - // printfn "%s" proc.Result.Error - - // test "exit code is 0 (Success)" { - // Expect.equal proc.ExitCode 0 $"incorrect exit code: {proc.Result.Error} | {proc.Result.Output}" - // } + ]) + testSequenced (testList "specification validation" [ + testSequenced (testList "latest" [ + // run: arc-validate validate -i fixtures/arcs/specification/v2.0.0-draft + // adapt this when a new latest specification package is available! + yield! + testFixture (Fixtures.withToolExecution + false + "../../../../../publish/arc-validate" + [|"--verbose"; "validate"; "-i"; "fixtures/arcs/specification/v2.0.0-draft"; "-o"; "."|] + (get_gh_api_token()) + ) [ + "Exit code is 0" , + fun tool args proc -> Expect.equal proc.ExitCode 0 (ErrorMessage.withProcessDiagnostics "incorrect exit code" proc tool args ) + "Console Output indicates that the tool will validate against specs" , + fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("running `arc-validate validate` without")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) + "Console Output indicates that the chosen spec version is latest" , + fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("Performing validation against version 'latest' of the ARC specification")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) + "Console Output indicates that latest spec version is mapped to a validation package correctly" , + fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("latest spec version supported is 2.0.0-draft")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) + "Ouptput files exist", + fun tool args proc -> + Expect.isTrue (Directory.Exists(".arc-validate-results")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/badge.svg")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/badge.svg does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/validation_report.xml")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/validation_report.xml does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/validation_summary.json")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/validation_summary.json does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + "Test arc passes spec validation", + fun tool args proc -> + let summary = + ".arc-validate-results/arc_specification@2.0.0/validation_summary.json" + |> File.ReadAllText + |> fun x -> x.ReplaceLineEndings("\n") + |> ValidationSummary.fromJson - // test "resultFileExists" { - // Expect.isTrue (File.Exists("fixtures/arcs/inveniotestarc/arc-validate-results.xml")) "result file does not exist at expected location" - // } + Expect.equal summary.Critical.Failed 0 (ErrorMessage.withProcessDiagnostics "incorrect number of critical failures" proc tool args ) + Expect.equal summary.Critical.Errored 0 (ErrorMessage.withProcessDiagnostics "incorrect number of critical errors" proc tool args ) + Expect.isFalse summary.Critical.HasFailures (ErrorMessage.withProcessDiagnostics "expected no critical failures" proc tool args ) + + Expect.equal summary.Critical.Failed 0 (ErrorMessage.withProcessDiagnostics "incorrect number of noncritical failures" proc tool args ) + Expect.equal summary.Critical.Errored 0 (ErrorMessage.withProcessDiagnostics "incorrect number of noncritical failures" proc tool args ) + Expect.isFalse summary.Critical.HasFailures (ErrorMessage.withProcessDiagnostics "expected no noncritical failures" proc tool args ) - // test "passed tests"{ - // let validationResults = ValidationResults.fromJUnitFile "fixtures/arcs/inveniotestarc/arc-validate-results.xml" - // Expect.equal - // validationResults.PassedTests - // ReferenceObjects.``invenio test arc validation results``.PassedTests - // "incorrect test results" - - // } + ] + ]) + testSequenced (testList "v2-0-0-draft" [ + // run: arc-validate validate --specification-version 2.0.0-draft -i fixtures/arcs/specification/v2.0.0-draft + yield! + testFixture (Fixtures.withToolExecution + false + "../../../../../publish/arc-validate" + [|"--verbose"; "validate"; "--specification-version"; "2.0.0-draft"; "-i"; "fixtures/arcs/specification/v2.0.0-draft"; "-o"; "."|] + (get_gh_api_token()) + ) [ + "Exit code is 0" , + fun tool args proc -> Expect.equal proc.ExitCode 0 (ErrorMessage.withProcessDiagnostics "incorrect exit code" proc tool args ) + "Console Output indicates that the tool will validate against specs" , + fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("running `arc-validate validate` without")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) + "Console Output indicates that the chosen spec version is correct" , + fun tool args proc -> Expect.isTrue (proc.Result.Output.Contains("Performing validation against version '2.0.0-draft' of the ARC specification")) (ErrorMessage.withProcessDiagnostics "incorrect console output" proc tool args ) + "Ouptput files exist", + fun tool args proc -> + Expect.isTrue (Directory.Exists(".arc-validate-results")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/badge.svg")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/badge.svg does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/validation_report.xml")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/validation_report.xml does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + Expect.isTrue (File.Exists(".arc-validate-results/arc_specification@2.0.0/validation_summary.json")) (ErrorMessage.withProcessDiagnostics $".arc-validate-results/arc_specification@2.0.0/validation_summary.json does not exist in {System.Environment.CurrentDirectory}" proc tool args ) + "Test arc passes spec validation", + fun tool args proc -> + let summary = + ".arc-validate-results/arc_specification@2.0.0/validation_summary.json" + |> File.ReadAllText + |> fun x -> x.ReplaceLineEndings("\n") + |> ValidationSummary.fromJson - // test "failed tests" { - // let validationResults = ValidationResults.fromJUnitFile "fixtures/arcs/inveniotestarc/arc-validate-results.xml" - // Expect.equal - // validationResults.FailedTests - // ReferenceObjects.``invenio test arc validation results``.FailedTests - // "incorrect test results" - // } + Expect.equal summary.Critical.Failed 0 (ErrorMessage.withProcessDiagnostics "incorrect number of critical failures" proc tool args ) + Expect.equal summary.Critical.Errored 0 (ErrorMessage.withProcessDiagnostics "incorrect number of critical errors" proc tool args ) + Expect.isFalse summary.Critical.HasFailures (ErrorMessage.withProcessDiagnostics "expected no critical failures" proc tool args ) + + Expect.equal summary.Critical.Failed 0 (ErrorMessage.withProcessDiagnostics "incorrect number of noncritical failures" proc tool args ) + Expect.equal summary.Critical.Errored 0 (ErrorMessage.withProcessDiagnostics "incorrect number of noncritical failures" proc tool args ) + Expect.isFalse summary.Critical.HasFailures (ErrorMessage.withProcessDiagnostics "expected no noncritical failures" proc tool args ) - // test "errored tests" { - // let validationResults = ValidationResults.fromJUnitFile "fixtures/arcs/inveniotestarc/arc-validate-results.xml" - // Expect.equal - // validationResults.ErroredTests - // ReferenceObjects.``invenio test arc validation results``.ErroredTests - // "incorrect test results" - // } - // ] - //) + ] + ]) + ]) ]) \ No newline at end of file diff --git a/tests/arc-validate.Tests/arc-validate.Tests.fsproj b/tests/arc-validate.Tests/arc-validate.Tests.fsproj index e5bce12..ef4bd96 100644 --- a/tests/arc-validate.Tests/arc-validate.Tests.fsproj +++ b/tests/arc-validate.Tests/arc-validate.Tests.fsproj @@ -1,4 +1,4 @@ - + Exe net8.0 @@ -23,5 +23,7 @@ + + \ No newline at end of file diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/README.md b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/README.md new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/dataset/.gitkeep b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/dataset/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.assay.xlsx b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.assay.xlsx new file mode 100644 index 0000000..4cae63e Binary files /dev/null and b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.assay.xlsx differ diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.dataset.xlsx b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.dataset.xlsx new file mode 100644 index 0000000..4acd55d Binary files /dev/null and b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/isa.dataset.xlsx differ diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/protocols/.gitkeep b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/assays/measurement1/protocols/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/isa.investigation.xlsx b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/isa.investigation.xlsx new file mode 100644 index 0000000..d14c611 Binary files /dev/null and b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/isa.investigation.xlsx differ diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/runs/FSharpArcCapsule.yml b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/runs/FSharpArcCapsule.yml new file mode 100644 index 0000000..2ed7449 --- /dev/null +++ b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/runs/FSharpArcCapsule.yml @@ -0,0 +1,22 @@ +arcDirectory: + class: Directory + path: ../ +firstArg: + class: File + path: ../assays/measurement1/dataset/table.csv +secondArg: ./result.csv + +arc:has process sequence: + - class: arc:process sequence + arc:has input: + - class: arc:data + arc:name: "./arc/assays/measurement1/dataset/table.csv" + arc:has output: + - class: arc:data + arc:name: "./arc/runs/fsResult1/result.csv" + +$namespaces: + arc: https://github.com/nfdi4plants/ARC_ontology + +$schemas: + - https://raw.githubusercontent.com/nfdi4plants/ARC_ontology/main/ARC_v2.0.owl diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/README.md b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/README.md new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/isa.study.xlsx b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/isa.study.xlsx new file mode 100644 index 0000000..fa4d6ad Binary files /dev/null and b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/isa.study.xlsx differ diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/protocols/.gitkeep b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/protocols/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/resources/.gitkeep b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/studies/experiment1_material/resources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/workflows/FixedScript/dummy.cwl b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/workflows/FixedScript/dummy.cwl new file mode 100644 index 0000000..e69de29 diff --git a/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/workflows/FixedScript/script.fsx b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/workflows/FixedScript/script.fsx new file mode 100644 index 0000000..5638d8b --- /dev/null +++ b/tests/arc-validate.Tests/fixtures/arcs/specification/v2.0.0-draft/workflows/FixedScript/script.fsx @@ -0,0 +1 @@ +"Hello" \ No newline at end of file