Merge pull request #609 from github/aeisenberg/use-default-tools

Remove a TODO and use defautl tools option

.github/workflows/pr-checks.yml

@@ -7,6 +7,7 @@ on:
   push:
     branches: [main, v1]
   pull_request:
+  workflow_dispatch:
 
 jobs:
   lint-js:
@@ -119,9 +120,6 @@ jobs:
       with:
         config-file: ".github/codeql/codeql-config-packaging.yml"
         languages: javascript
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
-
     - name: Build code
       shell: bash
       run: ./build.sh
@@ -164,8 +162,6 @@ jobs:
         config-file: ".github/codeql/codeql-config-packaging2.yml"
         languages: javascript
         packs: dsp-testing/codeql-pack1@0.0.4, dsp-testing/codeql-pack2
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
 
     - name: Build code
       shell: bash
@@ -209,8 +205,6 @@ jobs:
         config-file: ".github/codeql/codeql-config-packaging3.yml"
         packs: +dsp-testing/codeql-pack1@0.0.4
         languages: javascript
-        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
-        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
 
     - name: Build code
       shell: bash
@@ -234,6 +228,59 @@ jobs:
           exit 1
         fi
 
+  # Tests a split workflow where database construction and query execution happen in different steps
+  test-split-workflow:
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        config-file: ".github/codeql/codeql-config-packaging3.yml"
+        packs: +dsp-testing/codeql-pack1@0.0.4
+        languages: javascript
+        tools: latest
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        skip-queries: true
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert No Results
+      run: |
+        if [ "$(ls -A $RUNNER_TEMP/results)" ]; then
+          echo "Expected results directory to be empty after skipping query execution!"
+          exit 1
+        fi
+    - uses: ./../action/analyze
+      with:
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi
 
   # Identify the CodeQL tool versions to integration test against.
   check-codeql-versions:
@@ -241,6 +288,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       versions: ${{ steps.compare.outputs.versions }}
+      nightly-url: ${{ steps.get-url.outputs.nightly-url }}
 
     steps:
     - uses: actions/checkout@v2
@@ -266,22 +314,45 @@ jobs:
       with:
         tools: latest
         languages: javascript
-    - name: Compare default and latest CodeQL bundle versions
+    - name: Remove empty database
+      # allows us to run init a third time
+      run: |
+        rm -rf "$RUNNER_TEMP/codeql_databases"
+    - name: Get nightly release URL
+      id: get-url
+      env:
+        GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+      shell: bash
+      # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
+      run: |
+        export LATEST=`gh release list --repo dsp-testing/codeql-cli-nightlies -L 1 | cut -f 3`
+        echo "::set-output name=nightly-url::https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST/codeql-bundle.tar.gz"
+    - name: Init with a CodeQL bundle from a nightly release
+      id: init-nightly
+      uses: ./../action/init
+      with:
+        tools: ${{ steps.get-url.outputs.nightly-url }}
+        languages: javascript
+    - name: Compare CodeQL bundle versions
       id: compare
       env:
         CODEQL_DEFAULT: ${{ steps.init-default.outputs.codeql-path }}
         CODEQL_LATEST: ${{ steps.init-latest.outputs.codeql-path }}
+        CODEQL_NIGHTLY: ${{ steps.init-nightly.outputs.codeql-path }}
+        NIGHTLY_URL: ${{ steps.get-url.outputs.nightly-url }}
       run: |
         CODEQL_VERSION_DEFAULT="$("$CODEQL_DEFAULT" version --format terse)"
         CODEQL_VERSION_LATEST="$("$CODEQL_LATEST" version --format terse)"
+        CODEQL_VERSION_NIGHTLY="$("$CODEQL_NIGHTLY" version --format terse)"
         echo "Default CodeQL bundle version is $CODEQL_VERSION_DEFAULT"
         echo "Latest CodeQL bundle version is $CODEQL_VERSION_LATEST"
+        echo "Nightly CodeQL bundle version is $CODEQL_VERSION_NIGHTLY"
         if [[ "$CODEQL_VERSION_DEFAULT" == "$CODEQL_VERSION_LATEST" ]]; then
-          # Just use `tools: null` to avoid duplication in the integration tests.
+          # Skip `tools: latest` since it would be the same as `tools: null`
-          VERSIONS_JSON='[null]'
+          VERSIONS_JSON="[null, \"$NIGHTLY_URL\"]"
         else
-          # Use both `tools: null` and `tools: latest` in the integration tests.
+          # Run integration tests with all three bundles.
-          VERSIONS_JSON='[null, "latest"]'
+          VERSIONS_JSON="[null, \"$NIGHTLY_URL\", \"latest\"]"
         fi
         # Output a JSON-encoded list with the distinct versions to test against.
         echo "Suggested matrix config for integration tests: $VERSIONS_JSON"
@@ -803,3 +874,23 @@ jobs:
         # Deliberately don't use TEST_MODE here. This is specifically testing
         # the compatibility with the API.
         runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  multi-language-repo_test-local-codeql:
+    needs: [check-js, check-node-modules, check-codeql-versions]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      run: |
+        wget ${{ needs.check-codeql-versions.outputs.nightly-url }}
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        tools: ../action/codeql-bundle.tar.gz
+    - name: Build code
+      run: ./build.sh
+    - uses: ./../action/analyze

CHANGELOG.md

@@ -2,7 +2,14 @@
 
 ## [UNRELEASED]
 
-- Fix `RUNNER_TEMP environment variable must be set` when using runner.
+- The `analyze` step of the Action now supports a `skip-queries` option to merely build the CodeQL database without analyzing. This functionality is not present in the runner. Additionally, the step will no longer fail if it encounters a finalized database, and will instead continue with query execution. [#602](https://github.com/github/codeql-action/pull/602)
+- Update the warning message when the baseline lines of code count is unavailable. [#608](https://github.com/github/codeql-action/pull/608)
+
+## 1.0.4 - 28 Jun 2021
+
+- Fix `RUNNER_TEMP environment variable must be set` when using runner. [#594](https://github.com/github/codeql-action/pull/594)
+- Fix couting of lines of code for C# projects. [#586](https://github.com/github/codeql-action/pull/586)
+
 ## 1.0.3 - 23 Jun 2021
 
 No user facing changes.

README.md

@@ -128,3 +128,11 @@ By default, this will override any queries specified in a config file. If you wi
 ## Troubleshooting
 
 Read about [troubleshooting code scanning](https://help.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-code-scanning).
+
+### Note on "missing analysis" message
+
+The very first time code scanning is run and if it is on a pull request, you will probably get a message mentioning a "missing analysis". This is expected.
+
+After code scanning has analyzed the code in a pull request, it needs to compare the analysis of the topic branch (the merge commit of the branch you used to create the pull request) with the analysis of the base branch (the branch into which you want to merge the pull request). This allows code scanning to compute which alerts are newly introduced by the pull request, which alerts were already present in the base branch, and whether any existing alerts are fixed by the changes in the pull request. Initially, if you use a pull request to add code scanning to a repository, the base branch has not yet been analyzed, so it's not possible to compute these details. In this case, when you click through from the results check on the pull request you will see the "Missing analysis for base commit SHA-HASH" message.
+
+For more information and other causes of this message, see [Reasons for the "missing analysis" message](https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/setting-up-code-scanning-for-a-repository#reasons-for-the-missing-analysis-message)

analyze/action.yml

@@ -24,6 +24,10 @@ inputs:
     description: Specify whether or not to add code snippets to the output sarif file.
     required: false
     default: "false"
+  skip-queries:
+    description: If this option is set, the CodeQL database will be built but no queries will be run on it. Thus, no results will be produced.
+    required: false
+    default: "false"
   threads:
     description: The number of threads to be used by CodeQL.
     required: false

lib/analyze-action.js

@@ -34,7 +34,8 @@ async function sendStatusReport(startedAt, stats, error) {
 }
 async function run() {
     const startedAt = new Date();
-    let stats = undefined;
+    let uploadStats = undefined;
+    let runStats = undefined;
     let config = undefined;
     util.initializeEnvironment(util.Mode.actions, pkg.version);
     try {
@@ -51,7 +52,11 @@ async function run() {
             url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
         };
         const outputDir = actionsUtil.getRequiredInput("output");
-        const queriesStats = await analyze_1.runAnalyze(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger), actionsUtil.getOptionalInput("category"), config, logger);
+        const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger);
+        await analyze_1.runFinalize(outputDir, threads, config, logger);
+        if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
+            runStats = await analyze_1.runQueries(outputDir, util.getMemoryFlag(actionsUtil.getOptionalInput("ram")), util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
+        }
         if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
             await analyze_1.runCleanup(config, actionsUtil.getOptionalInput("cleanup-level") || "brutal", logger);
         }
@@ -60,13 +65,11 @@ async function run() {
             dbLocations[language] = util.getCodeQLDatabasePath(config, language);
         }
         core.setOutput("db-locations", dbLocations);
-        if (actionsUtil.getRequiredInput("upload") === "true") {
+        if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
-            const uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
+            uploadStats = await upload_lib.uploadFromActions(outputDir, config.gitHubVersion, apiDetails, logger);
-            stats = { ...queriesStats, ...uploadStats };
         }
         else {
             logger.info("Not uploading results");
-            stats = { ...queriesStats };
         }
         const repositoryNwo = repository_1.parseRepositoryNwo(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
         await database_upload_1.uploadDatabases(repositoryNwo, config, apiDetails, logger);
@@ -75,9 +78,12 @@ async function run() {
         core.setFailed(error.message);
         console.log(error);
         if (error instanceof analyze_1.CodeQLAnalysisError) {
-            stats = { ...error.queriesStatusReport };
+            const stats = { ...error.queriesStatusReport };
+            await sendStatusReport(startedAt, stats, error);
+        }
+        else {
+            await sendStatusReport(startedAt, undefined, error);
         }
-        await sendStatusReport(startedAt, stats, error);
         return;
     }
     finally {
@@ -103,7 +109,15 @@ async function run() {
             }
         }
     }
-    await sendStatusReport(startedAt, stats);
+    if (runStats && uploadStats) {
+        await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
+    }
+    else if (runStats) {
+        await sendStatusReport(startedAt, { ...runStats });
+    }
+    else {
+        await sendStatusReport(startedAt, undefined);
+    }
 }
 async function runWrapper() {
     try {

lib/analyze-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAKmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAqC,SAAS,CAAC;IACxD,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,YAAY,GAAG,MAAM,oBAAU,CACnC,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,EACpE,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;QAEF,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACrD,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CACpD,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;YACF,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;SAC7C;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACrC,KAAK,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;SAC7B;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;SAC1C;QAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAChD,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
+{"version":3,"file":"analyze-action.js","sourceRoot":"","sources":["../src/analyze-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,oDAAsC;AAEtC,4DAA8C;AAC9C,uCAMmB;AACnB,iDAAmD;AACnD,uDAAoD;AACpD,uCAA6C;AAC7C,6CAAkD;AAClD,yDAA2C;AAE3C,6CAA+B;AAE/B,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAUvC,KAAK,UAAU,gBAAgB,CAC7B,SAAe,EACf,KAAuC,EACvC,KAAa;;IAEb,MAAM,MAAM,GACV,OAAA,KAAK,0CAAE,wBAAwB,MAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,SAAS,CAAC;IAChB,MAAM,gBAAgB,GAAG,MAAM,WAAW,CAAC,sBAAsB,CAC/D,QAAQ,EACR,MAAM,EACN,SAAS,QACT,KAAK,0CAAE,OAAO,QACd,KAAK,0CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAAuB;QACvC,GAAG,gBAAgB;QACnB,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACjB,CAAC;IACF,MAAM,WAAW,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC;AACnD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,WAAW,GAAmC,SAAS,CAAC;IAC5D,IAAI,QAAQ,GAAoC,SAAS,CAAC;IAC1D,IAAI,MAAM,GAAuB,SAAS,CAAC;IAC3C,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3D,IAAI;QACF,IACE,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,CAClC,MAAM,WAAW,CAAC,sBAAsB,CACtC,QAAQ,EACR,UAAU,EACV,SAAS,CACV,CACF,CAAC,EACF;YACA,OAAO;SACR;QACD,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;QAClC,MAAM,GAAG,MAAM,wBAAS,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtE,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QAED,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,WAAW,CAAC,gBAAgB,CAAC,OAAO,CAAC;YAC3C,GAAG,EAAE,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC;SACnD,CAAC;QACF,MAAM,SAAS,GAAG,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CACjC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EACvC,MAAM,CACP,CAAC;QACF,MAAM,qBAAW,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,KAAK,MAAM,EAAE;YAC3D,QAAQ,GAAG,MAAM,oBAAU,CACzB,SAAS,EACT,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EACvD,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,EACrE,OAAO,EACP,WAAW,CAAC,gBAAgB,CAAC,UAAU,CAAC,EACxC,MAAM,EACN,MAAM,CACP,CAAC;SACH;QAED,IAAI,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,MAAM,EAAE;YAC5D,MAAM,oBAAU,CACd,MAAM,EACN,WAAW,CAAC,gBAAgB,CAAC,eAAe,CAAC,IAAI,QAAQ,EACzD,MAAM,CACP,CAAC;SACH;QAED,MAAM,WAAW,GAA+B,EAAE,CAAC;QACnD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SACtE;QACD,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;QAE5C,IAAI,QAAQ,IAAI,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,MAAM,EAAE;YACjE,WAAW,GAAG,MAAM,UAAU,CAAC,iBAAiB,CAC9C,SAAS,EACT,MAAM,CAAC,aAAa,EACpB,UAAU,EACV,MAAM,CACP,CAAC;SACH;aAAM;YACL,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;SACtC;QAED,MAAM,aAAa,GAAG,+BAAkB,CACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;QACF,MAAM,iCAAe,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;KAClE;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEnB,IAAI,KAAK,YAAY,6BAAmB,EAAE;YACxC,MAAM,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,mBAAmB,EAAE,CAAC;YAC/C,MAAM,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SACjD;aAAM;YACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;SACrD;QAED,OAAO;KACR;YAAS;QACR,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,MAAM,KAAK,SAAS,EAAE;YAC1C,IAAI,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YAC7D,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,MAAM,iBAAiB,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACvE,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;gBAE1D,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;oBACnC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;wBAC3B,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;4BAClB,IAAI,CAAC,UAAU,CACb,uBAAuB,QAAQ,MAAM,KAAK,CAAC,IAAI,EAAE,CAClD,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAC/C,CAAC;4BACF,IAAI,CAAC,QAAQ,EAAE,CAAC;yBACjB;6BAAM,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;4BAC9B,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;yBAC7C;qBACF;gBACH,CAAC,CAAC;gBACF,YAAY,CAAC,aAAa,CAAC,CAAC;aAC7B;SACF;KACF;IAED,IAAI,QAAQ,IAAI,WAAW,EAAE;QAC3B,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;KACpE;SAAM,IAAI,QAAQ,EAAE;QACnB,MAAM,gBAAgB,CAAC,SAAS,EAAE,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;KACpD;SAAM;QACL,MAAM,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;KAC9C;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/analyze.js

@@ -10,6 +10,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
+const yaml = __importStar(require("js-yaml"));
 const analysisPaths = __importStar(require("./analysis-paths"));
 const codeql_1 = require("./codeql");
 const count_loc_1 = require("./count-loc");
@@ -55,7 +56,8 @@ async function createdDBForScannedLanguages(config, logger) {
     analysisPaths.includeAndExcludeAnalysisPaths(config);
     const codeql = codeql_1.getCodeQL(config.codeQLCmd);
     for (const language of config.languages) {
-        if (languages_1.isScannedLanguage(language)) {
+        if (languages_1.isScannedLanguage(language) &&
+            !dbIsFinalized(config, language, logger)) {
             logger.startGroup(`Extracting ${language}`);
             if (language === languages_1.Language.python) {
                 await setupPythonExtractor(logger);
@@ -65,13 +67,29 @@ async function createdDBForScannedLanguages(config, logger) {
         }
     }
 }
+function dbIsFinalized(config, language, logger) {
+    const dbPath = util.getCodeQLDatabasePath(config, language);
+    try {
+        const dbInfo = yaml.load(fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8"));
+        return !("inProgress" in dbInfo);
+    }
+    catch (e) {
+        logger.warning(`Could not check whether database for ${language} was finalized. Assuming it is not.`);
+        return false;
+    }
+}
 async function finalizeDatabaseCreation(config, threadsFlag, logger) {
     await createdDBForScannedLanguages(config, logger);
     const codeql = codeql_1.getCodeQL(config.codeQLCmd);
     for (const language of config.languages) {
-        logger.startGroup(`Finalizing ${language}`);
+        if (dbIsFinalized(config, language, logger)) {
-        await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
+            logger.info(`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`);
-        logger.endGroup();
+        }
+        else {
+            logger.startGroup(`Finalizing ${language}`);
+            await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config, language), threadsFlag);
+            logger.endGroup();
+        }
     }
 }
 // Runs queries and creates sarif files in the given folder
@@ -183,15 +201,13 @@ function packWithVersionToQuerySuiteEntry(pack) {
     }
     return text;
 }
-async function runAnalyze(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
+async function runFinalize(outputDir, threadsFlag, config, logger) {
     // Delete the tracer config env var to avoid tracing ourselves
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
     fs.mkdirSync(outputDir, { recursive: true });
     await finalizeDatabaseCreation(config, threadsFlag, logger);
-    const queriesStats = await runQueries(outputDir, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger);
-    return { ...queriesStats };
 }
-exports.runAnalyze = runAnalyze;
+exports.runFinalize = runFinalize;
 async function runCleanup(config, cleanupLevel, logger) {
     logger.startGroup("Cleaning up databases");
     for (const language of config.languages) {

lib/codeql.js

@@ -154,50 +154,58 @@ async function setupCodeQL(codeqlURL, apiDetails, tempDir, toolCacheDir, variant
         if (forceLatest) {
             codeqlURL = undefined;
         }
-        const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
+        let codeqlFolder;
-        const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+        let codeqlURLVersion;
-        // If we find the specified version, we always use that.
+        if (codeqlURL && !codeqlURL.startsWith("http")) {
-        let codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
+            codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
-        // If we don't find the requested version, in some cases we may allow a
+            codeqlURLVersion = "local";
-        // different version to save download time if the version hasn't been
-        // specified explicitly (in which case we always honor it).
-        if (!codeqlFolder && !codeqlURL && !forceLatest) {
-            const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
-            if (codeqlVersions.length === 1) {
-                const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
-                if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-                    logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
-                    codeqlFolder = tmpCodeqlFolder;
-                }
-            }
-        }
-        if (codeqlFolder) {
-            logger.debug(`CodeQL found in cache ${codeqlFolder}`);
         }
         else {
-            if (!codeqlURL) {
+            codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
-                codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+            const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+            // If we find the specified version, we always use that.
+            codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
+            // If we don't find the requested version, in some cases we may allow a
+            // different version to save download time if the version hasn't been
+            // specified explicitly (in which case we always honor it).
+            if (!codeqlFolder && !codeqlURL && !forceLatest) {
+                const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
+                if (codeqlVersions.length === 1) {
+                    const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
+                    if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+                        logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
+                        codeqlFolder = tmpCodeqlFolder;
+                    }
+                }
             }
-            const parsedCodeQLURL = new URL(codeqlURL);
+            if (codeqlFolder) {
-            const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
+                logger.debug(`CodeQL found in cache ${codeqlFolder}`);
-            const headers = { accept: "application/octet-stream" };
-            // We only want to provide an authorization header if we are downloading
-            // from the same GitHub instance the Action is running on.
-            // This avoids leaking Enterprise tokens to dotcom.
-            // We also don't want to send an authorization header if there's already a token provided in the URL.
-            if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
-                parsedQueryString["token"] === undefined) {
-                logger.debug("Downloading CodeQL bundle with token.");
-                headers.authorization = `token ${apiDetails.auth}`;
             }
             else {
-                logger.debug("Downloading CodeQL bundle without token.");
+                if (!codeqlURL) {
+                    codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+                }
+                const parsedCodeQLURL = new URL(codeqlURL);
+                const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
+                const headers = { accept: "application/octet-stream" };
+                // We only want to provide an authorization header if we are downloading
+                // from the same GitHub instance the Action is running on.
+                // This avoids leaking Enterprise tokens to dotcom.
+                // We also don't want to send an authorization header if there's already a token provided in the URL.
+                if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
+                    parsedQueryString["token"] === undefined) {
+                    logger.debug("Downloading CodeQL bundle with token.");
+                    headers.authorization = `token ${apiDetails.auth}`;
+                }
+                else {
+                    logger.debug("Downloading CodeQL bundle without token.");
+                }
+                logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
+                const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
+                logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+                const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
+                codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
             }
-            logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
-            const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
-            logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-            const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
-            codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
         }
         let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
         if (process.platform === "win32") {

lib/count-loc.js

@@ -77,14 +77,13 @@ async function countLoc(cwd, include, exclude, dbLanguages, logger) {
         }
     }
     else {
-        logger.info("Could not determine the total number of lines of code in this repository. " +
+        logger.info("Could not determine the baseline lines of code count in this repository. " +
-            "Because of this, it will not be possible to compare the number of lines " +
+            "Because of this, it will not be possible to compare the lines " +
-            "of code analyzed by code scanning with the total number of lines of " +
+            "of code analyzed by code scanning with the baseline. This will not affect " +
-            "code in the repository. This will not affect the results produced by code " +
+            "the results produced by code scanning. If you have any questions, you can " +
-            "scanning. If you have any questions, you can raise an issue at " +
+            "raise an issue at https://github.com/github/codeql-action/issues. Please " +
-            "https://github.com/github/codeql-action/issues. Please include a link " +
+            "include a link to the repository if public, or otherwise information about " +
-            "to the repository if public, or otherwise information about the code scanning " +
+            "the code scanning workflow you are using.");
-            "workflow you are using.");
     }
     return lineCounts;
 }

lib/count-loc.js.map

@@ -1 +1 @@
-{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,4EAA4E;YAC1E,0EAA0E;YAC1E,sEAAsE;YACtE,4EAA4E;YAC5E,iEAAiE;YACjE,wEAAwE;YACxE,gFAAgF;YAChF,yBAAyB,CAC5B,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA/CD,4BA+CC"}
+{"version":3,"file":"count-loc.js","sourceRoot":"","sources":["../src/count-loc.ts"],"names":[],"mappings":";;AAAA,qDAAyC;AAEzC,2CAAuC;AAEvC,iCAAqC;AAKrC,sFAAsF;AACtF,MAAM,iBAAiB,GAA6B;IAClD,CAAC,EAAE,oBAAQ,CAAC,GAAG;IACf,KAAK,EAAE,oBAAQ,CAAC,GAAG;IACnB,IAAI,EAAE,oBAAQ,CAAC,MAAM;IACrB,EAAE,EAAE,oBAAQ,CAAC,EAAE;IACf,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;IAC/B,MAAM,EAAE,oBAAQ,CAAC,MAAM;IACvB,IAAI,EAAE,oBAAQ,CAAC,IAAI;IACnB,UAAU,EAAE,oBAAQ,CAAC,UAAU;CAChC,CAAC;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAC7D,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACd,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;KAChB;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,OAAO,GAAG,CAAC;AACb,CAAC,EACD,EAAgC,CACjC,CAAC;AAEF,SAAgB,WAAW,CAAC,QAAkB;IAC5C,QAAQ,QAAQ,EAAE;QAChB,KAAK,oBAAQ,CAAC,GAAG;YACf,OAAO,KAAK,CAAC;QACf,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,EAAE;YACd,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC;QAChB,KAAK,oBAAQ,CAAC,UAAU;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,MAAM;YAClB,OAAO,IAAI,CAAC;QACd,KAAK,oBAAQ,CAAC,IAAI;YAChB,OAAO,IAAI,CAAC;QAEd;YACE,kBAAW,CAAC,QAAQ,CAAC,CAAC;KACzB;AACH,CAAC;AApBD,kCAoBC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAiB,EACjB,OAAiB,EACjB,WAAuB,EACvB,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,IAAI,wBAAM,CAAC;QAC9B,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxE,OAAO;QACP,iBAAiB,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;KACvE,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,uDAAuD;IACvD,uDAAuD;IACvD,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CACxD,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,eAAe,IAAI,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE;YAC5D,GAAG,CAAC,eAAe,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;SAC3D;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAA8B,CAC/B,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QACrC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;YAC1D,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;SACzC;KACF;SAAM;QACL,MAAM,CAAC,IAAI,CACT,2EAA2E;YACzE,gEAAgE;YAChE,4EAA4E;YAC5E,4EAA4E;YAC5E,2EAA2E;YAC3E,6EAA6E;YAC7E,2CAA2C,CAC9C,CAAC;KACH;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA9CD,4BA8CC"}

lib/runner.js

@@ -243,7 +243,9 @@ program
             url: util_1.parseGitHubUrl(cmd.githubUrl),
         };
         const outputDir = cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-        await analyze_1.runAnalyze(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), util_1.getThreadsFlag(cmd.threads, logger), cmd.category, config, logger);
+        const threads = util_1.getThreadsFlag(cmd.threads, logger);
+        await analyze_1.runFinalize(outputDir, threads, config, logger);
+        await analyze_1.runQueries(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {
             logger.info("Not uploading results");
             return;

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

runner/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

runner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql-runner",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "private": true,
   "description": "CodeQL runner",
   "scripts": {

src/analyze-action.ts

@@ -5,16 +5,18 @@ import * as core from "@actions/core";
 
 import * as actionsUtil from "./actions-util";
 import {
-  runAnalyze,
   CodeQLAnalysisError,
   QueriesStatusReport,
   runCleanup,
+  runQueries,
+  runFinalize,
 } from "./analyze";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
+import { UploadStatusReport } from "./upload-lib";
 import * as util from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -53,7 +55,8 @@ async function sendStatusReport(
 
 async function run() {
   const startedAt = new Date();
-  let stats: AnalysisStatusReport | undefined = undefined;
+  let uploadStats: UploadStatusReport | undefined = undefined;
+  let runStats: QueriesStatusReport | undefined = undefined;
   let config: Config | undefined = undefined;
   util.initializeEnvironment(util.Mode.actions, pkg.version);
 
@@ -82,15 +85,22 @@ async function run() {
       url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
     };
     const outputDir = actionsUtil.getRequiredInput("output");
-    const queriesStats = await runAnalyze(
+    const threads = util.getThreadsFlag(
-      outputDir,
+      actionsUtil.getOptionalInput("threads"),
-      util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
-      util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
-      util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger),
-      actionsUtil.getOptionalInput("category"),
-      config,
       logger
     );
+    await runFinalize(outputDir, threads, config, logger);
+    if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
+      runStats = await runQueries(
+        outputDir,
+        util.getMemoryFlag(actionsUtil.getOptionalInput("ram")),
+        util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")),
+        threads,
+        actionsUtil.getOptionalInput("category"),
+        config,
+        logger
+      );
+    }
 
     if (actionsUtil.getOptionalInput("cleanup-level") !== "none") {
       await runCleanup(
@@ -106,17 +116,15 @@ async function run() {
     }
     core.setOutput("db-locations", dbLocations);
 
-    if (actionsUtil.getRequiredInput("upload") === "true") {
+    if (runStats && actionsUtil.getRequiredInput("upload") === "true") {
-      const uploadStats = await upload_lib.uploadFromActions(
+      uploadStats = await upload_lib.uploadFromActions(
         outputDir,
         config.gitHubVersion,
         apiDetails,
         logger
       );
-      stats = { ...queriesStats, ...uploadStats };
     } else {
       logger.info("Not uploading results");
-      stats = { ...queriesStats };
     }
 
     const repositoryNwo = parseRepositoryNwo(
@@ -128,10 +136,12 @@ async function run() {
     console.log(error);
 
     if (error instanceof CodeQLAnalysisError) {
-      stats = { ...error.queriesStatusReport };
+      const stats = { ...error.queriesStatusReport };
+      await sendStatusReport(startedAt, stats, error);
+    } else {
+      await sendStatusReport(startedAt, undefined, error);
     }
 
-    await sendStatusReport(startedAt, stats, error);
     return;
   } finally {
     if (core.isDebug() && config !== undefined) {
@@ -161,7 +171,13 @@ async function run() {
     }
   }
 
-  await sendStatusReport(startedAt, stats);
+  if (runStats && uploadStats) {
+    await sendStatusReport(startedAt, { ...runStats, ...uploadStats });
+  } else if (runStats) {
+    await sendStatusReport(startedAt, { ...runStats });
+  } else {
+    await sendStatusReport(startedAt, undefined);
+  }
 }
 
 async function runWrapper() {

src/analyze.ts

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
+import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import { getCodeQL } from "./codeql";
@@ -117,7 +118,10 @@ async function createdDBForScannedLanguages(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    if (isScannedLanguage(language)) {
+    if (
+      isScannedLanguage(language) &&
+      !dbIsFinalized(config, language, logger)
+    ) {
       logger.startGroup(`Extracting ${language}`);
 
       if (language === Language.python) {
@@ -133,6 +137,25 @@ async function createdDBForScannedLanguages(
   }
 }
 
+function dbIsFinalized(
+  config: configUtils.Config,
+  language: Language,
+  logger: Logger
+) {
+  const dbPath = util.getCodeQLDatabasePath(config, language);
+  try {
+    const dbInfo = yaml.load(
+      fs.readFileSync(path.resolve(dbPath, "codeql-database.yml"), "utf8")
+    );
+    return !("inProgress" in dbInfo);
+  } catch (e) {
+    logger.warning(
+      `Could not check whether database for ${language} was finalized. Assuming it is not.`
+    );
+    return false;
+  }
+}
+
 async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
@@ -142,12 +165,18 @@ async function finalizeDatabaseCreation(
 
   const codeql = getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
-    logger.startGroup(`Finalizing ${language}`);
+    if (dbIsFinalized(config, language, logger)) {
-    await codeql.finalizeDatabase(
+      logger.info(
-      util.getCodeQLDatabasePath(config, language),
+        `There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`
-      threadsFlag
+      );
-    );
+    } else {
-    logger.endGroup();
+      logger.startGroup(`Finalizing ${language}`);
+      await codeql.finalizeDatabase(
+        util.getCodeQLDatabasePath(config, language),
+        threadsFlag
+      );
+      logger.endGroup();
+    }
   }
 }
 
@@ -349,33 +378,18 @@ function packWithVersionToQuerySuiteEntry(
   return text;
 }
 
-export async function runAnalyze(
+export async function runFinalize(
   outputDir: string,
-  memoryFlag: string,
-  addSnippetsFlag: string,
   threadsFlag: string,
-  automationDetailsId: string | undefined,
   config: configUtils.Config,
   logger: Logger
-): Promise<QueriesStatusReport> {
+) {
   // Delete the tracer config env var to avoid tracing ourselves
   delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];
 
   fs.mkdirSync(outputDir, { recursive: true });
 
   await finalizeDatabaseCreation(config, threadsFlag, logger);
-
-  const queriesStats = await runQueries(
-    outputDir,
-    memoryFlag,
-    addSnippetsFlag,
-    threadsFlag,
-    automationDetailsId,
-    config,
-    logger
-  );
-
-  return { ...queriesStats };
 }
 
 export async function runCleanup(

src/codeql.ts

@@ -318,96 +318,101 @@ export async function setupCodeQL(
     if (forceLatest) {
       codeqlURL = undefined;
     }
-
+    let codeqlFolder: string;
-    const codeqlURLVersion = getCodeQLURLVersion(
+    let codeqlURLVersion: string;
-      codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
+    if (codeqlURL && !codeqlURL.startsWith("http")) {
-    );
+      codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
-    const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+      codeqlURLVersion = "local";
-
-    // If we find the specified version, we always use that.
-    let codeqlFolder = toolcache.find(
-      "CodeQL",
-      codeqlURLSemVer,
-      toolCacheDir,
-      logger
-    );
-
-    // If we don't find the requested version, in some cases we may allow a
-    // different version to save download time if the version hasn't been
-    // specified explicitly (in which case we always honor it).
-    if (!codeqlFolder && !codeqlURL && !forceLatest) {
-      const codeqlVersions = toolcache.findAllVersions(
-        "CodeQL",
-        toolCacheDir,
-        logger
-      );
-      if (codeqlVersions.length === 1) {
-        const tmpCodeqlFolder = toolcache.find(
-          "CodeQL",
-          codeqlVersions[0],
-          toolCacheDir,
-          logger
-        );
-        if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-          logger.debug(
-            `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
-          );
-          codeqlFolder = tmpCodeqlFolder;
-        }
-      }
-    }
-
-    if (codeqlFolder) {
-      logger.debug(`CodeQL found in cache ${codeqlFolder}`);
     } else {
-      if (!codeqlURL) {
+      codeqlURLVersion = getCodeQLURLVersion(
-        codeqlURL = await getCodeQLBundleDownloadURL(
+        codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
-          apiDetails,
+      );
-          variant,
+      const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
-          logger
-        );
-      }
 
-      const parsedCodeQLURL = new URL(codeqlURL);
+      // If we find the specified version, we always use that.
-      const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
+      codeqlFolder = toolcache.find(
-      const headers: IHeaders = { accept: "application/octet-stream" };
-      // We only want to provide an authorization header if we are downloading
-      // from the same GitHub instance the Action is running on.
-      // This avoids leaking Enterprise tokens to dotcom.
-      // We also don't want to send an authorization header if there's already a token provided in the URL.
-      if (
-        codeqlURL.startsWith(`${apiDetails.url}/`) &&
-        parsedQueryString["token"] === undefined
-      ) {
-        logger.debug("Downloading CodeQL bundle with token.");
-        headers.authorization = `token ${apiDetails.auth}`;
-      } else {
-        logger.debug("Downloading CodeQL bundle without token.");
-      }
-      logger.info(
-        `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
-      );
-      const codeqlPath = await toolcache.downloadTool(
-        codeqlURL,
-        tempDir,
-        headers
-      );
-      logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-
-      const codeqlExtracted = await toolcache.extractTar(
-        codeqlPath,
-        tempDir,
-        logger
-      );
-      codeqlFolder = await toolcache.cacheDir(
-        codeqlExtracted,
         "CodeQL",
         codeqlURLSemVer,
         toolCacheDir,
         logger
       );
-    }
 
+      // If we don't find the requested version, in some cases we may allow a
+      // different version to save download time if the version hasn't been
+      // specified explicitly (in which case we always honor it).
+      if (!codeqlFolder && !codeqlURL && !forceLatest) {
+        const codeqlVersions = toolcache.findAllVersions(
+          "CodeQL",
+          toolCacheDir,
+          logger
+        );
+        if (codeqlVersions.length === 1) {
+          const tmpCodeqlFolder = toolcache.find(
+            "CodeQL",
+            codeqlVersions[0],
+            toolCacheDir,
+            logger
+          );
+          if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+            logger.debug(
+              `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
+            );
+            codeqlFolder = tmpCodeqlFolder;
+          }
+        }
+      }
+
+      if (codeqlFolder) {
+        logger.debug(`CodeQL found in cache ${codeqlFolder}`);
+      } else {
+        if (!codeqlURL) {
+          codeqlURL = await getCodeQLBundleDownloadURL(
+            apiDetails,
+            variant,
+            logger
+          );
+        }
+
+        const parsedCodeQLURL = new URL(codeqlURL);
+        const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
+        const headers: IHeaders = { accept: "application/octet-stream" };
+        // We only want to provide an authorization header if we are downloading
+        // from the same GitHub instance the Action is running on.
+        // This avoids leaking Enterprise tokens to dotcom.
+        // We also don't want to send an authorization header if there's already a token provided in the URL.
+        if (
+          codeqlURL.startsWith(`${apiDetails.url}/`) &&
+          parsedQueryString["token"] === undefined
+        ) {
+          logger.debug("Downloading CodeQL bundle with token.");
+          headers.authorization = `token ${apiDetails.auth}`;
+        } else {
+          logger.debug("Downloading CodeQL bundle without token.");
+        }
+        logger.info(
+          `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
+        );
+        const codeqlPath = await toolcache.downloadTool(
+          codeqlURL,
+          tempDir,
+          headers
+        );
+        logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+
+        const codeqlExtracted = await toolcache.extractTar(
+          codeqlPath,
+          tempDir,
+          logger
+        );
+        codeqlFolder = await toolcache.cacheDir(
+          codeqlExtracted,
+          "CodeQL",
+          codeqlURLSemVer,
+          toolCacheDir,
+          logger
+        );
+      }
+    }
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";

src/count-loc.ts

@@ -98,14 +98,13 @@ export async function countLoc(
     }
   } else {
     logger.info(
-      "Could not determine the total number of lines of code in this repository. " +
+      "Could not determine the baseline lines of code count in this repository. " +
-        "Because of this, it will not be possible to compare the number of lines " +
+        "Because of this, it will not be possible to compare the lines " +
-        "of code analyzed by code scanning with the total number of lines of " +
+        "of code analyzed by code scanning with the baseline. This will not affect " +
-        "code in the repository. This will not affect the results produced by code " +
+        "the results produced by code scanning. If you have any questions, you can " +
-        "scanning. If you have any questions, you can raise an issue at " +
+        "raise an issue at https://github.com/github/codeql-action/issues. Please " +
-        "https://github.com/github/codeql-action/issues. Please include a link " +
+        "include a link to the repository if public, or otherwise information about " +
-        "to the repository if public, or otherwise information about the code scanning " +
+        "the code scanning workflow you are using."
-        "workflow you are using."
     );
   }
 

src/runner.ts

@@ -4,7 +4,7 @@ import * as path from "path";
 
 import { Command } from "commander";
 
-import { runAnalyze } from "./analyze";
+import { runFinalize, runQueries } from "./analyze";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
 import { CodeQL, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
@@ -431,11 +431,13 @@ program
 
       const outputDir =
         cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-      await runAnalyze(
+      const threads = getThreadsFlag(cmd.threads, logger);
+      await runFinalize(outputDir, threads, config, logger);
+      await runQueries(
         outputDir,
         getMemoryFlag(cmd.ram),
         getAddSnippetsFlag(cmd.addSnippets),
-        getThreadsFlag(cmd.threads, logger),
+        threads,
         cmd.category,
         config,
         logger