rework and fix Spreadsheet reader according to data node changes

src/Core/Process/Data.fs

@@ -1,31 +1,36 @@
-namespace ARCtrl.Process
+namespace ARCtrl
 
 open ARCtrl
+open ARCtrl.Process
 open ARCtrl.Helper 
 
 type Data = 
     {
         ID : URI option
         Name : string option
         DataType : DataFile option
+        Format : string option
+        SelectorFormat : URI option
         Comments : Comment list option
     }
 
-    static member make id name dataType comments =
+    static member make id name dataType format selectorFormat comments =
         {
             ID      = id
             Name    = name
             DataType = dataType
+            Format  = format
+            SelectorFormat = selectorFormat
             Comments = comments         
         }
 
-    static member create (?Id,?Name,?DataType,?Comments) = 
-        Data.make Id Name DataType Comments
+    static member create (?Id,?Name,?DataType,?Format,?SelectorFormat,?Comments) = 
+        Data.make Id Name DataType Format SelectorFormat Comments
 
     static member empty =
         Data.create()
 
-    member this.NameAsString =
+    member this.NameText =
         this.Name
         |> Option.defaultValue ""
 
@@ -35,5 +40,5 @@ type Data =
         member this.PrintCompact() =
             match this.DataType with
             | Some t ->
-                sprintf "%s [%s]" this.NameAsString t.AsString 
-            | None -> sprintf "%s" this.NameAsString
+                sprintf "%s [%s]" this.NameText t.AsString 
+            | None -> sprintf "%s" this.NameText

src/Core/Table/ArcTableAux.fs

@@ -114,7 +114,12 @@ module SanityChecks =
             let (ci,_),cell = en.Current.Key,en.Current.Value
             let header = headers.[ci]
             let headerIsFreetext = not header.IsTermColumn
+            let headerIsData = header.IsDataColumn
             let cellIsNotFreetext = not cell.isFreeText
+            let cellIsNotData = not cell.isData
+            if headerIsData && (cellIsNotData && cellIsNotFreetext) then 
+                (if raiseException then failwith else printfn "%s") $"Invalid combination of header `{header}` and cell `{cell}`. Data header should contain either Data or Freetext cells."
+                isValid <- false
             if headerIsFreetext && cellIsNotFreetext then 
                 (if raiseException then failwith else printfn "%s") $"Invalid combination of header `{header}` and cell `{cell}`."
                 isValid <- false

src/Core/Table/CompositeCell.fs

@@ -15,10 +15,12 @@ type CompositeCell =
     ///
     /// https://isa-specs.readthedocs.io/en/latest/isatab.html#unit
     | Unitized of string*OntologyAnnotation
+    | Data of Data
 
     member this.isUnitized = match this with | Unitized _ -> true | _ -> false
     member this.isTerm = match this with | Term _ -> true | _ -> false
     member this.isFreeText = match this with | FreeText _ -> true | _ -> false
+    member this.isData = match this with | Data _ -> true | _ -> false
 
     /// <summary>
     /// This returns the default empty cell from an existing CompositeCell.
@@ -28,6 +30,7 @@ type CompositeCell =
         | CompositeCell.Term _ -> CompositeCell.emptyTerm
         | CompositeCell.Unitized _ -> CompositeCell.emptyUnitized
         | CompositeCell.FreeText _ -> CompositeCell.emptyFreeText
+        | CompositeCell.Data _ -> CompositeCell.emptyData
 
     /// <summary>
     /// This function returns an array of all values as string
@@ -44,6 +47,7 @@ type CompositeCell =
         | FreeText s -> [|s|]
         | Term oa -> [| oa.NameText; defaultArg oa.TermSourceREF ""; defaultArg oa.TermAccessionNumber ""|]
         | Unitized (v,oa) -> [| v; oa.NameText; defaultArg oa.TermSourceREF ""; defaultArg oa.TermAccessionNumber ""|]
+        | Data d -> [| defaultArg d.Name ""; defaultArg d.Format ""; defaultArg d.SelectorFormat ""|]
 
     /// FreeText string will be converted to unit term name.
     ///
@@ -53,6 +57,7 @@ type CompositeCell =
         | Unitized _ -> this
         | FreeText text -> CompositeCell.Unitized ("", OntologyAnnotation.create(text))
         | Term term -> CompositeCell.Unitized ("", term)
+        | Data d -> failwith "Data cell cannot be converted to Unitized cell."
 
     /// FreeText string will be converted to term name.
     ///
@@ -62,13 +67,15 @@ type CompositeCell =
         | Term _ -> this
         | Unitized (_,unit) -> CompositeCell.Term unit
         | FreeText text -> CompositeCell.Term(OntologyAnnotation.create(text))
+        | Data d -> failwith "Data cell cannot be converted to Term cell."
 
     /// Will always keep `OntologyAnnotation.NameText` from Term or Unit.
     member this.ToFreeTextCell() =
         match this with
         | FreeText _ -> this
         | Term term -> FreeText(term.NameText)
         | Unitized (v,unit) -> FreeText(unit.NameText)
+        | Data d -> FreeText (Option.defaultValue "" d.Name)
 
     // Suggest this syntax for easy "of-something" access
     member this.AsUnitized  =
@@ -99,9 +106,15 @@ type CompositeCell =
 
     static member createFreeText (value: string) = FreeText value
 
+    static member createData (d:Data) = Data d
+
+    static member createDataFromString (value : string, ?format : string, ?selectorFormat : string) =
+        Data(Data.create(Name = value, ?Format = format, ?SelectorFormat = selectorFormat))
+
     static member emptyTerm = Term (OntologyAnnotation())
     static member emptyFreeText = FreeText ""
     static member emptyUnitized = Unitized ("", OntologyAnnotation())
+    static member emptyData = Data(Data.create())
 
     /// <summary>
     /// Updates current CompositeCell with information from OntologyAnnotation.
@@ -115,6 +128,7 @@ type CompositeCell =
         | CompositeCell.Term _ -> CompositeCell.createTerm oa
         | CompositeCell.Unitized (v,_) -> CompositeCell.createUnitized (v,oa)
         | CompositeCell.FreeText _ -> CompositeCell.createFreeText oa.NameText
+        | CompositeCell.Data d -> failwith "Data cell cannot be updated with OntologyAnnotation."
 
     /// <summary>
     /// Updates current CompositeCell with information from OntologyAnnotation.
@@ -132,6 +146,7 @@ type CompositeCell =
         | Term oa -> $"{oa.NameText}"
         | FreeText s -> s
         | Unitized (v,oa) -> $"{v} {oa.NameText}"
+        | Data d -> $"{d.NameText}"
 
 #if FABLE_COMPILER
     //[<CompiledName("Term")>]
@@ -143,5 +158,7 @@ type CompositeCell =
     //[<CompiledName("Unitized")>]
     static member unitized (v:string, oa:OntologyAnnotation) = CompositeCell.Unitized(v, oa)
 
+    //[<CompiledName("Data")>]
+    static member data (d:Process.Data) = CompositeCell.Data(d)
 #else
 #endif

src/Core/Table/CompositeColumn.fs

@@ -26,9 +26,17 @@ type CompositeColumn = {
         // no cell values will be handled later and is no error case
         | _, emptyCell when cells.Length = 0 -> 
             true
+        | isData when header.IsDataColumn && (cells.[0].isData || cells.[0].isFreeText) -> 
+            true
+        | isData when header.IsDataColumn -> 
+            if raiseExeption then 
+                let exampleCells = cells.[0]
+                let msg = $"Invalid combination of header `{header}` and cells `{exampleCells}`, Data header should have either Data or Freetext cells"
+                failwith msg
+            false
         | isTerm when header.IsTermColumn && (cells.[0].isTerm || cells.[0].isUnitized) -> 
             true
-        | isNotTerm when not header.IsTermColumn && cells.[0].isFreeText -> 
+        | isNotTerm when (not header.IsTermColumn) && cells.[0].isFreeText -> 
             true
         | h, c -> 
             if raiseExeption then 

src/Core/Table/CompositeHeader.fs

@@ -287,6 +287,11 @@ type CompositeHeader =
         | ProtocolType -> true 
         | anythingElse -> false
 
+    member this.IsDataColumn =
+        match this with 
+        | Input IOType.Data | Output IOType.Data -> true 
+        | anythingElse -> false
+
     /// <summary>
     /// Is true if the Building Block type is a FeaturedColumn. 
     ///

src/Json/Assay.fs

@@ -189,7 +189,7 @@ module AssayExtensions =
             Decode.fromJsonString Assay.ROCrate.decoder s
 
         /// exports in json-ld format
-        static member toROCrateJsonString(studyName, ?spaces) =
+        static member toROCrateJsonString(?studyName, ?spaces) =
             fun (obj: ArcAssay) ->
                 Assay.ROCrate.encoder studyName obj
                 |> Encode.toJsonString (Encode.defaultSpaces spaces)

src/Json/Process/Data.fs

@@ -22,6 +22,8 @@ module Data =
                 "@type", (Encode.list [Encode.string "Data"])
                 Encode.tryInclude "name" Encode.string (oa.Name)
                 Encode.tryInclude "type" DataFile.ROCrate.encoder oa.DataType
+                Encode.tryInclude "encodingFormat" Encode.string oa.Format
+                Encode.tryInclude "usageInfo" Encode.string oa.SelectorFormat
                 Encode.tryIncludeListOpt "comments" Comment.ROCrate.encoder oa.Comments
                 "@context", ROCrateContext.Data.context_jsonvalue
             ]
@@ -34,6 +36,8 @@ module Data =
                     ID = get.Optional.Field "@id" Decode.uri
                     Name = get.Optional.Field "name" Decode.string
                     DataType = get.Optional.Field "type" DataFile.ROCrate.decoder
+                    Format = get.Optional.Field "encodingFormat" Decode.string
+                    SelectorFormat = get.Optional.Field "usageInfo" Decode.uri
                     Comments = get.Optional.Field "comments" (Decode.list Comment.ROCrate.decoder)
                 }
 
@@ -60,6 +64,8 @@ module Data =
                     ID = get.Optional.Field "@id" Decode.uri
                     Name = get.Optional.Field "name" Decode.string
                     DataType = get.Optional.Field "type" DataFile.ISAJson.decoder
+                    Format = None
+                    SelectorFormat = None
                     Comments = get.Optional.Field "comments" (Decode.list Comment.ISAJson.decoder)
                 }
             )

src/Spreadsheet/ARCtrl.Spreadsheet.fsproj

@@ -19,8 +19,8 @@
     <Compile Include="Metadata\Assays.fs" />
     <Compile Include="Metadata\Study.fs" />
     <Compile Include="Metadata\OntologySourceReference.fs" />
-    <Compile Include="AnnotationTable\CompositeHeader.fs" />
     <Compile Include="AnnotationTable\CompositeCell.fs" />
+    <Compile Include="AnnotationTable\CompositeHeader.fs" />
     <Compile Include="AnnotationTable\CompositeColumn.fs" />
     <Compile Include="AnnotationTable\ArcTable.fs" />
     <Compile Include="ArcAssay.fs" />

src/Spreadsheet/AnnotationTable/ArcTable.fs

@@ -61,14 +61,21 @@ let classifyColumnOrder (column : CompositeColumn) =
 [<Literal>]
 let annotationTablePrefix = "annotationTable"
 
+let helperColumnStrings = 
+    [
+        "Term Source REF"
+        "Term Accession Number"
+        "Unit"
+        "Data Format"
+        "Data Selector Format"
+    ]
+
 let groupColumnsByHeader (columns : list<FsColumn>) = 
     columns
     |> Aux.List.groupWhen (fun c -> 
         let v = c.[1].ValueAsString()
-        Regex.tryParseReferenceColumnHeader v
-        |> Option.isNone
-        &&
-        (v.StartsWith "Unit" |> not)
+        List.exists (fun s -> v.StartsWith s) helperColumnStrings
+        |> not
     )
 
 /// Returns the annotation table of the worksheet if it exists, else returns None

src/Spreadsheet/AnnotationTable/CompositeCell.fs

@@ -4,14 +4,38 @@ open ARCtrl
 open ARCtrl.Helper
 open FsSpreadsheet
 
-let fromFsCells (cells : list<FsCell>) : CompositeCell =
+//let fromFsCells (cells : list<FsCell>) : CompositeCell =
+//    let cellValues = cells |> List.map (fun c -> c.ValueAsString())
+//    match cellValues with
+//    | [v] -> CompositeCell.createFreeText v
+//    | [v1;v2;v3] -> CompositeCell.createTermFromString(v1,v2,v3)
+//    | [v1;v2;v3;v4] -> CompositeCell.createUnitizedFromString(v1,v2,v3,v4)
+//    | _ -> 
+//        failwithf "Dafuq"
+
+let termFromFsCells (tsrCol : int option) (tanCol : int option ) (cells : list<FsCell>) : CompositeCell= 
+    let cellValues = cells |> List.map (fun c -> c.ValueAsString())
+    let tan = Option.map (fun i -> cellValues.[i]) tanCol
+    let tsr = Option.map (fun i -> cellValues.[i]) tsrCol
+    CompositeCell.createTermFromString(cellValues.[0],?tsr = tsr, ?tan = tan)
+
+let unitizedFromFsCells (unitCol : int) (tsrCol : int option ) (tanCol : int option) (cells : list<FsCell>) : CompositeCell =
+    let cellValues = cells |> List.map (fun c -> c.ValueAsString())
+    let unit = cellValues.[unitCol]
+    let tan = Option.map (fun i -> cellValues.[i]) tanCol
+    let tsr = Option.map (fun i -> cellValues.[i]) tsrCol
+    CompositeCell.createUnitizedFromString(cellValues.[0],unit,?tsr = tsr, ?tan = tan)
+
+let freeTextFromFsCells (cells : list<FsCell>) : CompositeCell =
+    let cellValues = cells |> List.map (fun c -> c.ValueAsString())
+    CompositeCell.createFreeText cellValues.[0]
+
+let dataFromFsCells (format : int option) (selectorFormat : int option) (cells : list<FsCell>) : CompositeCell =
     let cellValues = cells |> List.map (fun c -> c.ValueAsString())
-    match cellValues with
-    | [v] -> CompositeCell.createFreeText v
-    | [v1;v2;v3] -> CompositeCell.createTermFromString(v1,v2,v3)
-    | [v1;v2;v3;v4] -> CompositeCell.createUnitizedFromString(v1,v2,v3,v4)
-    | _ -> 
-        failwithf "Dafuq"
+    let format = Option.map (fun i -> cellValues.[i]) format
+    let selectorFormat = Option.map (fun i -> cellValues.[i]) selectorFormat
+    CompositeCell.createDataFromString(cellValues.[0],?format = format, ?selectorFormat = selectorFormat)
+
 
 let toFsCells isTerm hasUnit (cell : CompositeCell) : list<FsCell> =
     match cell with

src/Spreadsheet/AnnotationTable/CompositeColumn.fs

@@ -22,7 +22,7 @@ let fixDeprecatedIOHeader (col : FsColumn) =
         col
 
 let fromFsColumns (columns : list<FsColumn>) : CompositeColumn =
-    let header = 
+    let header, cellParser = 
         columns
         |> List.map (fun c -> c.[1])
         |> CompositeHeader.fromFsCells
@@ -32,7 +32,7 @@ let fromFsColumns (columns : list<FsColumn>) : CompositeColumn =
         for i = 2 to l do
             columns
             |> List.map (fun c -> c.[i])
-            |> CompositeCell.fromFsCells
+            |> cellParser
         |]                 
     CompositeColumn.create(header,cells)