Integration of Metaschema M4 features into OSCAL repository

.circleci/config.yml

@@ -79,21 +79,6 @@ commands:
           name: Get Saxon-HE
           command: |
             mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION
-      - run:
-          name: Build JSON Validation CLI
-          command: |
-            # update maven version
-            cd /opt
-            sudo wget --no-check-certificate "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
-            sudo tar -xvzf "apache-maven-${MAVEN_VERSION}-bin.tar.gz"
-            sudo mv "apache-maven-${MAVEN_VERSION}" maven
-            export M2_HOME=/opt/maven
-            export PATH=${M2_HOME}/bin:${PATH}
-            cd -
-            # now build
-            cd "${OSCAL_TOOLS_DIR}/json-cli"
-            /opt/maven/bin/mvn dependency:go-offline
-            /opt/maven/bin/mvn install
       - save_cache:
           paths:
             - ~/.m2
@@ -130,13 +115,13 @@ commands:
           name: Get xmllint
           command: |
             sudo apt-get install libxml2-utils
-  #install-npm-ajv:
-  #  description: "Install ajv"
-  #  steps:
-  #    - run:
-  #        name: Get ajv
-  #        command: |
-  #          sudo npm install -g ajv-cli
+  install-npm-ajv:
+    description: "Install ajv"
+    steps:
+      - run:
+          name: Get ajv
+          command: |
+            sudo npm install -g ajv-cli
   install-npm-json-diff:
     description: "Install json-diff"
     steps:
@@ -259,6 +244,7 @@ jobs: # a collection of
       - install-maven-dependencies
       - apt-update
       - install-apt-xmllint
+      - install-npm-ajv
       - run:
           name: Generate OSCAL schemas
           command: |
@@ -453,15 +439,16 @@ jobs: # a collection of
       - check-skip-build
       - apt-update
       - install-apt-xmllint
+      - install-npm-ajv
       - install-maven-dependencies
       - run:
           name: Run Content Unit Tests
           command: |
             bash "$CICD_DIR/run-unittests.sh" -w "$OSCAL_BUILD_DIR"
-      - run:
-          name: Run Profile Resolution Unit Tests
-          command: |
-            bash "$CICD_DIR/test-profile-resolution.sh" -w "$OSCAL_BUILD_DIR"
+#      - run:
+#          name: Run Profile Resolution Unit Tests
+#          command: |
+#            bash "$CICD_DIR/test-profile-resolution.sh" -w "$OSCAL_BUILD_DIR"
 workflows:
   version: 2
   build:

.gitmodules

@@ -6,4 +6,5 @@
 [submodule "build/metaschema"]
 	path = build/metaschema
 	url = https://github.com/usnistgov/metaschema.git
-	branch = master
+	branch = metaschema-m4-integration
+	update = rebase

CONTRIBUTING.md

@@ -32,8 +32,8 @@ The core OSCAL project team regularly reviews the open issues, prioritizes their
 
 There are two mailing lists for the project:
 
-- *[email protected]* for communication among parties interested in contributing to the development of OSCAL or exchanging ideas. Subscribe by sending an email to [oscal-dev-join@nist.gov](mailto:oscal-dev-join@nist.gov). To unsubscribe send an email to [oscal-dev-leave@nist.gov](mailto:oscal-dev-leave@nist.gov).
-- *[email protected]* for low-frequency updates on the status of the OSCAL project. Subscribe by sending an email to [oscal-updates-join@nist.gov](mailto:oscal-updates-join@nist.gov). To unsubscribe send an email to [oscal-updates-leave@nist.gov](mailto:oscal-updates-leave@nist.gov).
+- *[email protected]* for communication among parties interested in contributing to the development of OSCAL or exchanging ideas. Subscribe by sending an email to [oscal-dev+subscribe@list.nist.gov](mailto:oscal-dev+subscribe@list.nist.gov). To unsubscribe send an email to [oscal-dev+unsubscribe@list.nist.gov](mailto:oscal-dev+unsubscribe@list.nist.gov).
+- *oscal-updates@list.nist.gov* for low-frequency updates on the status of the OSCAL project. Subscribe by sending an email to [oscal-updates+subscribe@list.nist.gov](mailto:oscal-updates+subscribe@list.nist.gov). To unsubscribe send an email to [oscal-updates+unsubscribe@list.nist.gov](mailto:oscal-updates+unsubscribe@list.nist.gov).
 
 ## Contributing to the OSCAL repository
 
@@ -56,7 +56,7 @@ This repository consists of the following directories and files pertaining to th
 - [docs](docs): Stores sources for the [OSCAL website](https://pages.nist.gov/OSCAL).
 - [content](content): This directory once stored XML, JSON, and YAML content examples. These examples have been moved to the OSCAL content GitHub [repository](https://github.com/usnistgov/oscal-content/tree/master).
 - [build](build): Contains a collection of scripts, eXtensible Stylesheet Language Transformations (XSLTs), and other artifacts used to support this repository's continuous integration and continuous deployment (CI/CD) processes.
-- [src](src): This directory once stored source artifacts used to produce the XML, JSON, and YAML resources that were provided in this repository. These resources have been moved to the OSCAL content GitHub [repository](https://github.com/usnistgov/oscal-content/tree/master/src).
+- [src](src): This includes the resources used by the NIST OSCAL Team to manage OSCAL development, including the metaschema files used to maintain the OSCAL models across all supported formats. This directory once stored source artifacts used to produce the XML, JSON, and YAML resources that were provided in this repository. These resources have been moved to the OSCAL content GitHub [repository](https://github.com/usnistgov/oscal-content/tree/master/src).
 - [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md): This file contains a code of conduct for OSCAL project contributors.
 - [CONTRIBUTING.md](CONTRIBUTING.md): This file is for potential contributors to the OSCAL project. It provides basic information on the OSCAL project, describes the main ways people can make contributions, explains how to report issues with OSCAL, and lists pointers to additional sources of information. It also has instructions on establishing a development environment for contributing to the OSCAL project and using GitHub project cards to track development sprints.
 - [LICENSE.md](LICENSE.md): This file contains license and copyright information for the files in the OSCAL GitHub repository.

build/Dockerfile

@@ -2,8 +2,9 @@
 FROM ubuntu:18.04
 
 ARG saxonversion
+ARG hugoversion
 
-RUN apt-get update && apt-get install -y apt-utils libxml2-utils jq maven hugo nodejs npm build-essential python-pip git && apt-get clean
+RUN apt-get update && apt-get install -y wget apt-utils libxml2-utils jq maven nodejs npm build-essential python-pip git && apt-get clean
 RUN npm install -g prettyjson markdown-link-check json-diff ajv-cli
 RUN pip install lxml
 
@@ -12,6 +13,9 @@ RUN pip install lxml
 
 RUN mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=${saxonversion}
 
+RUN wget https://github.com/gohugoio/hugo/releases/download/v${hugoversion}/hugo_extended_${hugoversion}_Linux-64bit.deb
+RUN dpkg -i hugo_extended_${hugoversion}_Linux-64bit.deb
+
 #RUN chown -R user:user /home/user
 
 VOLUME ["/oscal"]

build/ci-cd/copy-and-convert-content.sh

@@ -29,41 +29,41 @@ Usage: $0 [options]
 EOF
 }
 
-OPTS=`getopt -o w:o:c:a:vh --long resolve-profiles,working-dir:,artifact-dir:,oscal-dir:,config-file:,help -n "$0" -- "$@"`
+OPTS=`getopt -o a:o:w:c:hv --long resolve-profiles,artifact-dir:,oscal-dir:,working-dir:,config-file:,help -n "$0" -- "$@"`
 if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; usage ; exit 1 ; fi
 
 # Process arguments
 eval set -- "$OPTS"
 while [ $# -gt 0 ]; do
   arg="$1"
   case "$arg" in
-    -w|--working-dir)
-      WORKING_DIR="$(realpath "$2")"
-      shift # past path
-      ;;
     --resolve-profiles)
       RESOLVE_PROFILES=true
       shift # past path
       ;;
+    -a|--artifact-dir)
+      ARTIFACT_DIR="$(realpath "$2")"
+      shift # past path
+      ;;
     -o|--oscal-dir)
       OSCAL_DIR="$(realpath "$2")"
       shift # past path
       ;;
-    -c|--config-file)
-      CONFIG_FILE="$(realpath "$2")"
+    -w|--working-dir)
+      WORKING_DIR="$(realpath "$2")"
       shift # past path
       ;;
-    -a|--artifact-dir)
-      ARTIFACT_DIR="$(realpath "$2")"
+    -c|--config-file)
+      CONFIG_FILE="$(realpath "$2")"
       shift # past path
       ;;
-    -v)
-      VERBOSE=true
-      ;;
     -h|--help)
       usage
       exit 0
       ;;
+    -v)
+      VERBOSE=true
+      ;;
     --) # end of options
       shift
       break;
@@ -95,7 +95,8 @@ CATALOG_SCHEMA="$(get_abs_path "${OSCAL_DIR}/xml/schema/oscal_catalog_schema.xsd
 
 # check for perl
 result=$(which perl 2>&1)
-if [ $? -ne 0 ]; then
+cmd_exitcode=$?
+if [ $cmd_exitcode != 0 ]; then
   echo -e "${P_ERROR}Perl is not installed. Perl is needed by this script.${P_END}"
   exit 1
 fi
@@ -169,7 +170,8 @@ post_process_content() {
       echo -e "${P_INFO}Producing pretty JSON '${P_END}${target_file_pretty_relative}${P_INFO}'.${P_END}"
     fi
     result=$(jq . "$target_file" > "$target_file_pretty" 2>&1)
-    if [ $? -ne 0 ]; then
+    cmd_exitcode=$?
+    if [ $cmd_exitcode != 0 ]; then
       echo -e "${P_ERROR}${result}${P_END}"
       echo -e "${P_ERROR}Unable to execute jq on '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}"
       return 1;
@@ -179,7 +181,8 @@ post_process_content() {
     perl -pi -e 's,\r,,g' "$target_file_pretty"
 
     result=$(validate_content "$target_file_pretty" "json" "$model" "$oscal_dir")
-    if [ $? -ne 0 ]; then
+    cmd_exitcode=$?
+    if [ $cmd_exitcode != 0 ]; then
       echo -e "${P_ERROR}${result}${P_END}"
       echo -e "${P_ERROR}Unable to validate content '${P_END}${target_file_pretty_relative}${P_ERROR}'.${P_END}"
       return 1;
@@ -290,7 +293,7 @@ copy_or_convert_content() {
     fi
     result=$(cp "$source_file" "$target_file" 2>&1)
     cmd_exitcode=$?
-    if [ $cmd_exitcode -ne 0 ]; then
+    if [ $cmd_exitcode != 0 ]; then
       echo -e "${P_ERROR}${result}${P_END}"
       echo -e "${P_ERROR}Unable to copy '${P_END}${source_file_relative}${P_ERROR}' to '${P_END}${target_file_relative}${P_ERROR}'.${P_END}"
       return 1;
@@ -303,11 +306,11 @@ copy_or_convert_content() {
     fi
 
     result=$(convert_to_format_and_validate "$source_file" "$target_file" "$source_format" "$target_format" "$model" "$oscal_dir")
+    cmd_exitcode=$?
     if [ -n "$result" ]; then
       echo -e "${result}"
     fi
-    cmd_exitcode=$?
-    if [ $cmd_exitcode -ne 0 ]; then
+    if [ $cmd_exitcode != 0 ]; then
       return 1;
     else
       echo -e "${P_OK}Converted ${source_format^^} ${model} '${P_END}${source_file_relative}${P_OK}' to ${target_format^^} as '${P_END}${target_file_relative}${P_OK}'.${P_END}"
@@ -319,10 +322,10 @@ copy_or_convert_content() {
     echo -e "${P_INFO}Post processing ${target_format^^} content '${P_END}${target_file_relative}${P_INFO}'.${P_END}"
   fi
   result=$(post_process_content $source_format $target_format $target_file $working_dir $oscal_dir)
+  cmd_exitcode=$?
   if [ -n "$result" ]; then
     echo -e "${result}"
   fi
-  cmd_exitcode=$?
   if [ $cmd_exitcode != 0 ]; then
     return 1;
   fi
@@ -341,7 +344,7 @@ copy_or_convert_content() {
 
         result=$(xsl_transform "${PROFILE_RESOLVER}" "$source_file" "${resolved_profile}" 2>&1)
         cmd_exitcode=$?
-        if [ $cmd_exitcode -ne 0 ]; then
+        if [ $cmd_exitcode != 0 ]; then
           if [ -n "$result" ]; then
             echo -e "${P_ERROR}${result}${P_END}"
           fi
@@ -350,7 +353,8 @@ copy_or_convert_content() {
         fi
 
         result=$(validate_xml "$CATALOG_SCHEMA" "${resolved_profile}")
-        if [ $cmd_exitcode -ne 0 ]; then
+        cmd_exitcode=$?
+        if [ $cmd_exitcode != 0 ]; then
           if [ -n "$result" ]; then
             echo -e "${P_ERROR}${result}${P_END}"
           fi
@@ -366,10 +370,10 @@ copy_or_convert_content() {
           echo -e "${P_INFO}Converting resolved profile '${P_END}${resolved_profile}${P_INFO}' to JSON.${P_END}"
         fi
         result="$(copy_or_convert_content "$source_dir" "$resolved_profile" "$working_dir" $src_format "catalog" $target_format "$working_dir" "$oscal_dir")"
+        cmd_exitcode=$?
         if [ -n "$result" ]; then
           echo -e "${result}"
         fi
-        cmd_exitcode=$?
         if [ $cmd_exitcode != 0 ]; then
           return 1;
         fi
@@ -392,7 +396,7 @@ process_paths() {
 #  printf 'models: %s\n' "${models[@]}"
 #  printf 'converttoformats: %s\n' "${conversion_formats[@]}"
 
-  local cmd_exitcode=0;
+  local exitcode=0;
   #shopt -s nullglob
   #shopt -s globstar
 
@@ -410,10 +414,10 @@ process_paths() {
 #    printf 'converttoformats: %s\n' "${converttoformats[@]}"
 
     result="$(copy_or_convert_content "$ARTIFACT_DIR" "$src_file" "${ARTIFACT_DIR}/src" $src_format $model $src_format "$WORKING_DIR" "$OSCAL_DIR")"
+    cmd_exitcode=$?
     if [ -n "$result" ]; then
       echo -e "${result}"
     fi
-    cmd_exitcode=$?
     if [ $cmd_exitcode -ne 0 ]; then
         exitcode=1
         continue;
@@ -430,18 +434,18 @@ process_paths() {
       fi
 
       result="$(copy_or_convert_content "$ARTIFACT_DIR" "$src_file" "${ARTIFACT_DIR}/src" $src_format $model $to_format "$WORKING_DIR" "$OSCAL_DIR")"
+      cmd_exitcode=$?
       if [ -n "$result" ]; then
         echo -e "${result}"
       fi
-      cmd_exitcode=$?
       if [ $cmd_exitcode -ne 0 ]; then
           exitcode=1
           continue;
       fi
 
     done
   done
-  return $cmd_exitcode;
+  return $exitcode;
 }
 
 process_paths #"${paths[@]}" "${formats[@]}" "${models[@]}" "${conversion_formats[@]}"