Merge pull request #1 from github/main

up to head

.codeqlmanifest.json

@@ -1,11 +1,27 @@
-{ "provide": [ "ruby/.codeqlmanifest.json",
-                "*/ql/src/qlpack.yml",
-               "*/ql/lib/qlpack.yml",
-               "*/ql/test/qlpack.yml",
-               "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
-               "*/ql/examples/qlpack.yml",
-               "*/upgrades/qlpack.yml",
-               "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
-               "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
-               "misc/legacy-support/*/qlpack.yml",
-               "misc/suite-helpers/qlpack.yml" ] }
+{
+    "provide": [
+        "*/ql/src/qlpack.yml",
+        "*/ql/lib/qlpack.yml",
+        "*/ql/test/qlpack.yml",
+        "*/ql/examples/qlpack.yml",
+        "*/ql/consistency-queries/qlpack.yml",
+        "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
+        "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/src/qlpack.yml",
+        "csharp/ql/campaigns/Solorigate/test/qlpack.yml",
+        "misc/legacy-support/*/qlpack.yml",
+        "misc/suite-helpers/qlpack.yml",
+        "ruby/extractor-pack/codeql-extractor.yml",
+        "ql/extractor-pack/codeql-extractor.yml"
+    ],
+    "versionPolicies": {
+      "default": {
+        "requireChangeNotes": true,
+        "committedPrereleaseSuffix": "dev",
+        "committedVersion": "nextPatchRelease"
+      }
+    }
+}

.gitattributes

@@ -50,4 +50,15 @@
 *.pdb -text
 
 java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+
+# For some languages, upgrade script testing references really old dbscheme
+# files from legacy upgrades that have CRLF line endings. Since upgrade
+# resolution relies on object hashes, we must suppress line ending conversion
+# for those testing dbscheme files.
+*/ql/lib/upgrades/initial/*.dbscheme -text
+
+# Generated test files - these are synced from the standard JavaScript libraries using
+# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
+javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/actions/fetch-codeql/action.yml

@@ -8,7 +8,7 @@ runs:
       run: |
         LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
         gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
-        unzip -q codeql-linux64.zip
-        echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
+        unzip -q -d "${RUNNER_TEMP}" codeql-linux64.zip
+        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/labeler.yml

@@ -18,7 +18,14 @@ Python:
   - python/**/*
   - change-notes/**/*python*
 
+Ruby:
+  - ruby/**/*
+  - change-notes/**/*ruby*
+
 documentation:
   - "**/*.qhelp"
   - "**/*.md"
   - docs/**/*
+
+"QL-for-QL": 
+  - ql/**/*

.github/workflows/check-change-note.yml

@@ -6,7 +6,11 @@ on:
     paths:
       - "*/ql/src/**/*.ql"
       - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
+      - "!ql/**"
+      - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:

.github/workflows/codeql-analysis.yml

@@ -27,6 +27,11 @@ jobs:
       pull-requests: read
 
     steps:
+    - name: Setup dotnet
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: 6.0.101
+
     - name: Checkout repository
       uses: actions/checkout@v2
 
@@ -51,7 +56,7 @@ jobs:
     #    uses a compiled language
 
     - run: |
-       dotnet build csharp
+       dotnet build csharp /p:UseSharedCompilation=false
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-metrics.yml

@@ -0,0 +1,43 @@
+name: "Publish framework coverage as metrics"
+
+on:
+  schedule:
+    - cron: '5 0 * * *'
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/csv-coverage-metrics.yml"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Create empty database
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          PROJECT="${{ runner.temp }}/java-project"
+          mkdir -p "$PROJECT/src/tmp/empty"
+          echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
+          codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
+      - name: Capture coverage information
+        run: |
+          DATABASE="${{ runner.temp }}/java-database"
+          codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
+      - uses: actions/upload-artifact@v2
+        with:
+          name: metrics.sarif
+          path: metrics.sarif
+          retention-days: 20
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v1
+        with:
+          sarif_file: metrics.sarif

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,76 @@
+name: JS ML-powered queries tests
+
+on:
+  push:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "javascript/ql/experimental/adaptivethreatmodeling/**"
+      - .github/workflows/js-ml-tests.yml
+
+defaults:
+  run:
+    working-directory: javascript/ql/experimental/adaptivethreatmodeling
+
+jobs:
+  qlformat:
+    name: Check QL formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Check QL formatting
+        run: |
+          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
+            xargs -0 codeql query format --check-only
+
+  qlcompile:
+    name: Check QL compilation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: |
+          for pack in modelbuilding src; do
+            codeql pack install --mode verify -- "${pack}"
+          done
+
+      - name: Check QL compilation
+        run: |
+          codeql query compile \
+            --check-only \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            --threads=0 \
+            -- \
+            lib modelbuilding src
+
+  qltest:
+    name: Run QL tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./.github/actions/fetch-codeql
+
+      - name: Install pack dependencies
+        run: codeql pack install -- test
+
+      - name: Run QL tests
+        run: |
+          codeql test run \
+            --threads=0 \
+            --ram 5120 \
+            --additional-packs "${{ github.workspace }}" \
+            -- \
+            test

.github/workflows/mad_modelDiff.yml

@@ -0,0 +1,103 @@
+name: Models as Data - Diff
+
+on:
+  workflow_dispatch:
+    inputs:
+      projects:
+        description: "The projects to generate models for"
+        required: true
+        default: '["netty/netty"]'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "java/ql/src/utils/model-generator/**/*.*"
+      - ".github/workflows/mad_modelDiff.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  model-diff:
+    name: Model Difference
+    runs-on: ubuntu-latest
+    if: github.repository == 'github/codeql'
+    strategy:
+      matrix:
+        slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
+    steps:
+      - name: Clone github/codeql from PR
+        uses: actions/checkout@v2
+        if: github.event.pull_request
+        with:
+          path: codeql-pr
+      - name: Clone github/codeql from main
+        uses: actions/checkout@v2
+        with:
+          path: codeql-main
+          ref: main
+      - uses: ./codeql-main/.github/actions/fetch-codeql
+      - name: Download database
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          set -x
+          mkdir lib-dbs
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
+          curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
+          unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
+          mkdir "lib-dbs/$SHORTNAME/"
+          mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
+      - name: Generate Models (PR and main)
+        run: |
+          set -x
+          mkdir tmp-models
+          MODELS=`pwd`/tmp-models
+          DATABASES=`pwd`/lib-dbs
+
+          analyzeDatabaseWithCheckout() {
+            QL_VARIANT=$1
+            DATABASE=$2
+            cd codeql-$QL_VARIANT
+            SHORTNAME=`basename $DATABASE`
+            python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
+            mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
+            cd ..
+          }
+
+          for d in $DATABASES/*/ ; do
+            ls -1 "$d"
+
+            analyzeDatabaseWithCheckout "main" $d
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
+            then
+              analyzeDatabaseWithCheckout "pr" $d
+            fi
+          done
+      - name: Install diff2html
+        if: github.event.pull_request
+        run: |
+          npm install -g diff2html-cli
+      - name: Generate Model Diff
+        if: github.event.pull_request
+        run: |
+          set -x
+          MODELS=`pwd`/tmp-models
+          ls -1 tmp-models/
+          for m in $MODELS/*_main.qll ; do
+            t="${m/main/"pr"}"
+            basename=`basename $m`
+            name="diff_${basename/_main.qll/""}"
+            (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
+          done
+      - uses: actions/upload-artifact@v2
+        with:
+          name: models
+          path: tmp-models/*.qll
+          retention-days: 20
+      - uses: actions/upload-artifact@v2
+        with:
+          name: diffs
+          path: tmp-models/*.html
+          retention-days: 20