Merge branch 'develop' into 801-queries-for-schema-documentation-top-…

…down-polish
usnistgov · Nov 1, 2022 · ad8a743 · ad8a743
2 parents 189cdc9 + 8a951dc
commit ad8a743
Show file tree

Hide file tree

Showing 19 changed files with 369 additions and 312 deletions.
diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml
@@ -11,15 +11,15 @@ jobs:
     if: github.event.action == 'opened'
     steps:
       - name: Add New Issues to Issue Triage Board
-        uses: peter-evans/create-or-update-project-card@866533e2ae61f13c35fd1d374049713fab43c729
+        uses: peter-evans/create-or-update-project-card@5eacbbd224b7814354861b555cc18a8359e2cebe
         with:
           project-name: Issue Triage
           column-name: Needs Triage
   removeCard:
     runs-on: ubuntu-20.04
     if: github.event.action == 'closed'
     steps:
-      - uses: alex-page/github-project-automation-plus@bb266ff4dde9242060e2d5418e120a133586d488
+      - uses: alex-page/github-project-automation-plus@1f8873e97e3c8f58161a323b7c568c1f623a1c4d
         with:
           project: Issue Triage
           column: Done

diff --git a/.github/workflows/package-release.yml b/.github/workflows/package-release.yml
@@ -38,7 +38,7 @@ jobs:
           bash "${OSCAL_CICD_PATH}/package-release.sh" "${OSCAL_WORKING_PATH}"
           tar -jcvf "${OSCAL_WORKING_PATH}/${RELEASE_NAME}.tar.bz2" -C "${OSCAL_WORKING_PATH}/archive" .
           (cd "${OSCAL_WORKING_PATH}/archive" && zip -r "../${RELEASE_NAME}.zip" .)
-      - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+      - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
         with:
           name: release-artifacts
           path: |

diff --git a/.github/workflows/workflow-generate-metaschema-resources.yml b/.github/workflows/workflow-generate-metaschema-resources.yml
@@ -124,7 +124,7 @@ jobs:
       run: |
         zip ${{ runner.temp }}/schematron-validations.zip -r ${{ runner.temp }} build/metaschema/toolchains/xslt-M4/validate/metaschema-composition-check-compiled.xsl
       working-directory: ${{ env.CHECKOUT_PATH }}
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: schemas-and-converters
         path: |
@@ -142,7 +142,7 @@ jobs:
     - name: Publish Schemas and Converters
       # only do this on master
       if: github.event.inputs.commit_resources == 'true' || inputs.commit_resources == true
-      uses: stefanzweifel/git-auto-commit-action@fd157da78fa13d9383e5580d1fd1184d89554b51
+      uses: stefanzweifel/git-auto-commit-action@0049e3fa4059ca715255fbbcb7dea4516f02ce0a
       with:
         repository: ${{ env.CHECKOUT_PATH }}
         file_pattern: xml json

diff --git a/.github/workflows/workflow-generate-website-reference.yml b/.github/workflows/workflow-generate-website-reference.yml
@@ -105,7 +105,7 @@ jobs:
         echo "/home/runner/go/bin" >> $GITHUB_PATH
     - name: Setup Golang
       if: steps.cache-hugo.outputs.cache-hit != 'true'
-      uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f
+      uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f
       with:
         go-version-file: '${{ env.BRANCH_PATH }}/build/go.mod'
         cache: true
@@ -149,7 +149,7 @@ jobs:
       run: |
         zip ${{ runner.temp }}/metaschema-website.zip -r public/
       working-directory: ${{ env.MAIN_PATH }}/docs
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: website
         path: |
@@ -158,7 +158,7 @@ jobs:
     - name: Publish Generated Pages
       # only do this on master
       if: github.event.inputs.commit_resources == 'true' || inputs.commit_resources == true
-      uses: stefanzweifel/git-auto-commit-action@fd157da78fa13d9383e5580d1fd1184d89554b51
+      uses: stefanzweifel/git-auto-commit-action@0049e3fa4059ca715255fbbcb7dea4516f02ce0a
       with:
         repository: ${{ env.MAIN_PATH }}
         file_pattern: docs

diff --git a/.github/workflows/workflow-generate-website.yml b/.github/workflows/workflow-generate-website.yml
@@ -96,7 +96,7 @@ jobs:
     - name: Generate specification documentation
       run: |
         bash "${CICD_PATH}/generate-specification-documentation.sh"
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: specification-docs
         path: |
@@ -115,7 +115,7 @@ jobs:
         echo "/home/runner/go/bin" >> $GITHUB_PATH
     - name: Setup Golang
       if: steps.cache-hugo.outputs.cache-hit != 'true'
-      uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f
+      uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f
       with:
         go-version-file: '${{ env.BUILD_PATH }}/go.mod'
         cache: true
@@ -133,7 +133,7 @@ jobs:
       run: |
         zip ${{ runner.temp }}/metaschema-website.zip -r public/
       working-directory: ${{ github.workspace }}/docs
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: website
         path: |
@@ -148,7 +148,7 @@ jobs:
         output: html-link-report.md
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+    - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
       with:
         name: html-link-report
         path: html-link-report.md
@@ -164,7 +164,7 @@ jobs:
         script: |
           core.setFailed('Link checker detected broken or invalid links, read attached report.')
     - name: Deploy Website (using access_token)
-      uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305
+      uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935
       if: github.event_name == 'push' && inputs.commit_resources == true && github.ref_name == 'main'
       with:
         github_token: ${{ secrets.access_token }}
@@ -173,7 +173,7 @@ jobs:
         publish_branch: nist-pages
         commit_message: Deploying website [ci deploy skip]
     - name: Deploy Website (using COMMIT_TOKEN)
-      uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305
+      uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935
       if: github.event_name == 'workflow_dispatch' && github.event.inputs.commit_resources == 'true' && github.ref_name == 'main'
       with:
         github_token: ${{ secrets.COMMIT_TOKEN }}

diff --git a/build/ci-cd/validate-content.sh b/build/ci-cd/validate-content.sh
@@ -6,6 +6,7 @@ if [ -z ${OSCAL_SCRIPT_INIT+x} ]; then
     source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)/include/init-oscal.sh"
 fi
 source "$OSCALDIR/build/metaschema/scripts/include/init-validate-content.sh"
+source "$OSCALDIR/build/metaschema/scripts/include/init-schematron.sh"
 
 # Option defaults
 ARTIFACT_DIR="${OSCALDIR}"
@@ -23,10 +24,16 @@ Usage: $0 [options]
 -o DIR, --oscal-dir DIR           OSCAL schema are located in DIR.
 -h, --help                        Display help
 -v                                Provide verbose output
+--scratch-dir DIR                 Generate temporary artifacts in DIR
+                                  If not provided a new directory will be
+                                  created under \$TMPDIR if set or in /tmp.
+--keep-temp-scratch-dir           If a scratch directory is automatically
+                                  created, it will not be automatically removed.
 EOF
 }
 
-if ! OPTS=$(getopt -o o:vhc:a: --long artifact-dir:,oscal-dir:,help,config-file: -n "$0" -- "$@"); then echo "Failed parsing options." >&2 ; usage ; exit 1 ; fi
+OPTS=$(getopt -o o:vhc:a: --long artifact-dir:,oscal-dir:,help,config-file:,scratch-dir:,keep-temp-scratch-dir -n "$0" -- "$@")
+if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; usage ; exit 1 ; fi
 
 # Process arguments
 eval set -- "$OPTS"
@@ -45,6 +52,13 @@ while [ $# -gt 0 ]; do
       ARTIFACT_DIR="$(realpath "$2")"
       shift # past path
       ;;
+    --scratch-dir)
+      SCRATCH_DIR="$(realpath "$2")"
+      shift # past path
+      ;;
+    --keep-temp-scratch-dir)
+      KEEP_TEMP_SCRATCH_DIR=true
+    ;;
     -v)
       VERBOSE=true
       ;;
@@ -76,6 +90,28 @@ if [ "$VERBOSE" = "true" ]; then
   echo -e "${P_INFO}Using config file:${P_END} ${CONFIG_FILE}"
 fi
 
+if [ -z "${SCRATCH_DIR+x}" ]; then
+  SCRATCH_DIR="$(mktemp -d)"
+  if [ "$KEEP_TEMP_SCRATCH_DIR" != "true" ]; then
+    function CleanupScratchDir() {
+      rc=$?
+      if [ "$VERBOSE" = "true" ]; then
+        echo -e ""
+        echo -e "${P_INFO}Cleanup${P_END}"
+        echo -e "${P_INFO}=======${P_END}"
+        echo -e "${P_INFO}Deleting scratch directory:${P_END} ${SCRATCH_DIR}"
+      fi
+      rm -rf "${SCRATCH_DIR}"
+      exit $rc
+    }
+    trap CleanupScratchDir EXIT
+  fi
+fi
+
+profile_schematron="oscal/src/utils/schematron/oscal-profile.sch"
+compiled_profile_schematron="${SCRATCH_DIR}/oscal-profile.xsl"
+build_schematron "${profile_schematron}" "${compiled_profile_schematron}"
+
 exitcode=0
 shopt -s nullglob
 shopt -s globstar
@@ -113,6 +149,21 @@ while IFS="|" read path format model converttoformats || [ -n "$path" ]; do
           else
             echo -e "${P_OK}XML Schema validation passed for '${P_END}${file_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
           fi
+
+          if [ "${model}" == "profile" ]; then
+            echo -e "${P_INFO}Validating profile with Schematron for project's requirements and recommendations.${P_INFO}${P_END}"
+            target_file=$(basename -- "${file_relative}")
+            svrl_result="/tmp/${target_file}.svrl"
+            result=$(validate_with_schematron "${SCRATCH_DIR}/oscal-profile.xsl" "${file_relative}" "$svrl_result" 2>&1)
+            cmd_exitcode=$?
+            if [ $cmd_exitcode -ne 0 ]; then
+              echo -e "${P_ERROR}Profile validation execution for '${P_END}${file_relative}${P_ERROR}' with Schematron '${P_END}${profile_schematron}${P_ERROR}' did not complete.${P_END}"
+              echo -e "${P_ERROR}${result}${P_END}"
+              exitcode=1
+            else
+              echo -e "${P_OK}Profile validation execution for '${P_END}${file_relative}${P_OK}' with Schematron '${P_END}${profile_schematron}${P_OK}' completed successfully.${P_END}"
+            fi
+          fi
         ;;
       json)
           schema="$WORKING_DIR/json/schema/oscal_${model}_schema.json"

diff --git a/build/go.mod b/build/go.mod
@@ -2,7 +2,7 @@ module github.com/usnistgov/OSCAL/build
 
 go 1.18
 
-require github.com/gohugoio/hugo v0.104.3
+require github.com/gohugoio/hugo v0.105.0
 
 require (
 	cloud.google.com/go v0.101.0 // indirect
@@ -47,10 +47,10 @@ require (
 	github.com/disintegration/gift v1.2.1 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/evanw/esbuild v0.15.9 // indirect
+	github.com/evanw/esbuild v0.15.12 // indirect
 	github.com/frankban/quicktest v1.14.3 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/getkin/kin-openapi v0.103.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/getkin/kin-openapi v0.107.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.5 // indirect
@@ -69,21 +69,22 @@ require (
 	github.com/googleapis/go-type-adapters v1.0.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hairyhenderson/go-codeowners v0.2.3-0.20201026200250-cdc7c0759690 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/invopop/yaml v0.1.0 // indirect
 	github.com/jdkato/prose v1.2.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kyokomi/emoji/v2 v2.2.10 // indirect
-	github.com/magefile/mage v1.13.0 // indirect
+	github.com/magefile/mage v1.14.0 // indirect
 	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/marekm4/color-extractor v1.2.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/muesli/smartcrop v0.3.0 // indirect
 	github.com/niklasfasching/go-org v1.6.5 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
@@ -94,24 +95,24 @@ require (
 	github.com/sanity-io/litter v1.5.5 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/cobra v1.5.0 // indirect
+	github.com/spf13/cobra v1.6.1 // indirect
 	github.com/spf13/fsync v0.9.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tdewolff/minify/v2 v2.12.1 // indirect
-	github.com/tdewolff/parse/v2 v2.6.3 // indirect
-	github.com/yuin/goldmark v1.4.15 // indirect
+	github.com/tdewolff/minify/v2 v2.12.4 // indirect
+	github.com/tdewolff/parse/v2 v2.6.4 // indirect
+	github.com/yuin/goldmark v1.5.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	gocloud.dev v0.24.0 // indirect
 	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
 	golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/net v0.1.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.11 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sys v0.1.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/tools v0.2.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/api v0.76.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect