Compare commits

..

6 Commits

Author SHA1 Message Date
Michael B. Gale 7e36708483 Convert check-js.sh to TypeScript 2026-07-31 17:17:54 +01:00
Michael B. Gale fccc166683 Move subprocess helpers from update-release-branch.ts to command.ts 2026-07-31 14:38:04 +01:00
Michael B. Gale f8f91fb336 Move check-codescanning-config/index.ts to pr-checks and add tests 2026-07-31 14:38:03 +01:00
Michael B. Gale 58f39a7d42 Convert check-sarif/index.js and add tests 2026-07-31 14:01:48 +01:00
Michael B. Gale 40b3f0c1bc Move upgrade-bundle.ts to pr-checks and add tests 2026-07-31 14:01:48 +01:00
Michael B. Gale 6e32e882a9 Add DEFAULTS_FILE constant 2026-07-31 14:01:48 +01:00
40 changed files with 1469 additions and 1654 deletions
@@ -54,16 +54,15 @@ runs:
env:
CODEQL_ACTION_TEST_MODE: 'true'
- name: Install dependencies
shell: bash
run: npm install --location=global ts-node js-yaml
- name: Check config
working-directory: ${{ github.action_path }}
shell: bash
env:
EXPECTED_CONFIG_FILE_CONTENTS: '${{ inputs.expected-config-file-contents }}'
run: ts-node ./index.ts "$RUNNER_TEMP/user-config.yaml" "$EXPECTED_CONFIG_FILE_CONTENTS"
run: |
npx tsx ../action/pr-checks/check-cs-config.ts \
--file "$RUNNER_TEMP/user-config.yaml" \
--expected-contents "$EXPECTED_CONFIG_FILE_CONTENTS"
- name: Clean up
shell: bash
if: always()
@@ -1,49 +0,0 @@
import * as core from '@actions/core'
import * as yaml from 'js-yaml'
import * as fs from 'fs'
import * as assert from 'assert'
const actualConfig = loadActualConfig()
function sortConfigArrays(config) {
for (const key of Object.keys(config)) {
const value = config[key];
if (key === 'queries' && Array.isArray(value)) {
config[key] = value.sort();
}
}
return config;
}
const rawExpectedConfig = process.argv[3].trim()
if (!rawExpectedConfig) {
core.setFailed('No expected configuration provided')
} else {
core.startGroup('Expected generated user config')
core.info(yaml.dump(JSON.parse(rawExpectedConfig)))
core.endGroup()
}
const expectedConfig = rawExpectedConfig ? JSON.parse(rawExpectedConfig) : undefined;
assert.deepStrictEqual(
sortConfigArrays(actualConfig),
sortConfigArrays(expectedConfig),
'Expected configuration does not match actual configuration'
);
function loadActualConfig() {
if (!fs.existsSync(process.argv[2])) {
core.info('No configuration file found')
return undefined
} else {
const rawActualConfig = fs.readFileSync(process.argv[2], 'utf8')
core.startGroup('Actual generated user config')
core.info(rawActualConfig)
core.endGroup()
return yaml.load(rawActualConfig)
}
}
+20 -2
View File
@@ -16,5 +16,23 @@ inputs:
Comma separated list of query ids that should NOT be included in this SARIF file.
runs:
using: node24
main: index.js
using: "composite"
steps:
- name: Run `check-sarif.ts`
shell: bash
env:
SARIF_FILE: ${{ inputs.sarif-file }}
QUERIES_RUN: ${{ inputs.queries-run }}
QUERIES_NOT_RUN: ${{ inputs.queries-not-run }}
run: |
if [[ -d pr-checks ]]; then
npx tsx ./pr-checks/check-sarif.ts \
--sarif-file "$SARIF_FILE" \
--queries-run "$QUERIES_RUN" \
--queries-not-run "$QUERIES_NOT_RUN"
else
npx tsx ../action/pr-checks/check-sarif.ts \
--sarif-file "$SARIF_FILE" \
--queries-run "$QUERIES_RUN" \
--queries-not-run "$QUERIES_NOT_RUN"
fi
-43
View File
@@ -1,43 +0,0 @@
'use strict'
const core = require('@actions/core')
const fs = require('fs')
const sarif = JSON.parse(fs.readFileSync(core.getInput('sarif-file'), 'utf8'))
const rules = sarif.runs[0].tool.extensions.flatMap(ext => ext.rules || [])
const ruleIds = rules.map(rule => rule.id)
// Check that all the expected queries ran
const expectedQueriesRun = getQueryIdsInput('queries-run')
const queriesThatShouldHaveRunButDidNot = expectedQueriesRun.filter(queryId => !ruleIds.includes(queryId))
if (queriesThatShouldHaveRunButDidNot.length > 0) {
core.setFailed(`The following queries were expected to run but did not: ${queriesThatShouldHaveRunButDidNot.join(', ')}`)
}
// Check that all the unexpected queries did not run
const expectedQueriesNotRun = getQueryIdsInput('queries-not-run')
const queriesThatShouldNotHaveRunButDid = expectedQueriesNotRun.filter(queryId => ruleIds.includes(queryId))
if (queriesThatShouldNotHaveRunButDid.length > 0) {
core.setFailed(`The following queries were NOT expected to have run but did: ${queriesThatShouldNotHaveRunButDid.join(', ')}`)
}
core.startGroup('All queries run')
rules.forEach(rule => {
core.info(`${rule.id}: ${(rule.properties && rule.properties.name) || rule.name}`)
})
core.endGroup()
core.startGroup('Full SARIF')
core.info(JSON.stringify(sarif, null, 2))
core.endGroup()
function getQueryIdsInput(name) {
return core.getInput(name)
.split(',')
.map(q => q.trim())
.filter(q => q.length > 0)
}
-14
View File
@@ -1,14 +0,0 @@
name: Update default CodeQL bundle
description: Updates 'src/defaults.json' to point to a new CodeQL bundle release.
runs:
using: composite
steps:
- name: Install ts-node
shell: bash
run: npm install -g ts-node
- name: Run update script
working-directory: ${{ github.action_path }}
shell: bash
run: ts-node ./index.ts
-67
View File
@@ -1,67 +0,0 @@
import * as fs from 'fs';
import * as github from '@actions/github';
interface BundleInfo {
bundleVersion: string;
cliVersion: string;
}
interface Defaults {
bundleVersion: string;
cliVersion: string;
priorBundleVersion: string;
priorCliVersion: string;
}
function getCodeQLCliVersionForRelease(release): string {
// We do not currently tag CodeQL bundles based on the CLI version they contain.
// Instead, we use a marker file `cli-version-<version>.txt` to record the CLI version.
// This marker file is uploaded as a release asset for all new CodeQL bundles.
const cliVersionsFromMarkerFiles = release.assets
.map((asset) => asset.name.match(/cli-version-(.*)\.txt/)?.[1])
.filter((v) => v)
.map((v) => v as string);
if (cliVersionsFromMarkerFiles.length > 1) {
throw new Error(
`Release ${release.tag_name} has multiple CLI version marker files.`
);
} else if (cliVersionsFromMarkerFiles.length === 0) {
throw new Error(
`Failed to find the CodeQL CLI version for release ${release.tag_name}.`
);
}
return cliVersionsFromMarkerFiles[0];
}
async function getBundleInfoFromRelease(release): Promise<BundleInfo> {
return {
bundleVersion: release.tag_name,
cliVersion: getCodeQLCliVersionForRelease(release)
};
}
async function getNewDefaults(currentDefaults: Defaults): Promise<Defaults> {
const release = github.context.payload.release;
console.log('Updating default bundle as a result of the following release: ' +
`${JSON.stringify(release)}.`)
const bundleInfo = await getBundleInfoFromRelease(release);
return {
bundleVersion: bundleInfo.bundleVersion,
cliVersion: bundleInfo.cliVersion,
priorBundleVersion: currentDefaults.bundleVersion,
priorCliVersion: currentDefaults.cliVersion
};
}
async function main() {
const previousDefaults: Defaults = JSON.parse(fs.readFileSync('../../../src/defaults.json', 'utf8'));
const newDefaults = await getNewDefaults(previousDefaults);
// Update the source file in the repository. Calling workflows should subsequently rebuild
// the Action to update `lib/defaults.json`.
fs.writeFileSync('../../../src/defaults.json', JSON.stringify(newDefaults, null, 2) + "\n");
}
// Ideally, we'd await main() here, but that doesn't work well with `ts-node`.
// So instead we rely on the fact that Node won't exit until the event loop is empty.
main();
@@ -63,7 +63,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Java
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
java-version: ${{ inputs.java-version || '17' }}
distribution: temurin
+1 -1
View File
@@ -63,7 +63,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Install Java
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
with:
java-version: ${{ inputs.java-version || '17' }}
distribution: temurin
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
npm ci
- name: Verify compiled JS up to date
run: .github/workflows/script/check-js.sh
run: npx tsx pr-checks/check-js.ts
- name: Run unit tests
if: always()
-33
View File
@@ -1,33 +0,0 @@
#!/bin/bash
set -eu
# Sanity check that repo is clean to start with
if [ ! -z "$(git status --porcelain)" ]; then
# If we get a fail here then this workflow needs attention...
>&2 echo "Failed: Repo should be clean before testing!"
exit 1
fi
# Wipe the lib directory in case there are extra unnecessary files in there
rm -rf lib
# Generate the JavaScript files
npm run-script build
# Check that repo is still clean
if [ ! -z "$(git status --porcelain)" ]; then
# If we get a fail here then the PR needs attention
>&2 echo "Failed: JavaScript files are not up to date. Run 'rm -rf lib && npm run-script build' to update"
git status
echo "### Transpiled JS diff" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
git diff --output="$RUNNER_TEMP/js.diff"
cat "$RUNNER_TEMP/js.diff" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
# Reset bundled files to allow other checks to test for changes
git checkout lib
# Fail this check
exit 1
fi
echo "Success: JavaScript files are up to date"
+1 -1
View File
@@ -50,7 +50,7 @@ jobs:
run: npm ci
- name: Update bundle
uses: ./.github/actions/update-bundle
run: npx tsx pr-checks/update-bundle.ts
- name: Set up CodeQL CLI from new bundle
id: setup-codeql
@@ -38,7 +38,7 @@ jobs:
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: github/enterprise-releases
token: ${{ secrets.CODEQL_CI_ENTERPRISE_RELEASE_PAT }}
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
path: ${{ github.workspace }}/enterprise-releases/
sparse-checkout: releases.json
-8
View File
@@ -6,14 +6,6 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
No user facing changes.
## 4.37.6 - 04 Aug 2026
- Changed the default filepath for the new remote file address format that was introduced in CodeQL Action 4.37.0 / 3.37.0 to `.github/codeql-config.yml` to align it with the suggested path that is used elsewhere. [#4070](https://github.com/github/codeql-action/pull/4070)
## 4.37.5 - 03 Aug 2026
- Fixed a bug where a network error while streaming the download of the CodeQL bundle could terminate the `init` Action instead of falling back to downloading the bundle before extracting it. [#4061](https://github.com/github/codeql-action/pull/4061)
## 4.37.4 - 29 Jul 2026
- This version of the CodeQL Action adds support for the `tools` input for the `codeql-action/init` step to be specified using a `github-codeql-tools` [repository property](https://docs.github.com/en/organizations/managing-organization-settings/managing-custom-properties-for-repositories-in-your-organization). This feature will gradually be rolled out following the release of this version. Once rolled out, this allows for the CodeQL CLI version that is used in GitHub-managed workflows, such as Default Setup, to be set to a custom value. For example, customers who run into issues with rate limits when a new CodeQL CLI version is released can set the value to `toolcache` to always use the CodeQL CLI version that is available in the runner toolcache. For Advanced Setup workflows, the value provided for `tools` in the workflow definition always takes precedence unless the value of the repository property starts with `!`. [#4037](https://github.com/github/codeql-action/pull/4037)
+559 -862
View File
File diff suppressed because it is too large Load Diff
+31 -31
View File
@@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "4.37.7",
"version": "4.37.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "4.37.7",
"version": "4.37.5",
"license": "MIT",
"workspaces": [
"pr-checks"
@@ -31,7 +31,7 @@
"follow-redirects": "^1.16.0",
"get-folder-size": "^5.0.0",
"https-proxy-agent": "^7.0.6",
"js-yaml": "^5.2.2",
"js-yaml": "^5.2.1",
"jsonschema": "1.5.0",
"long": "^5.3.2",
"node-forge": "^1.4.0",
@@ -61,7 +61,7 @@
"eslint-plugin-jsdoc": "^62.9.0",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^13.0.6",
"globals": "^17.8.0",
"globals": "^17.7.0",
"nock": "^14.0.16",
"sinon": "^22.1.0",
"typescript": "^6.0.3",
@@ -374,9 +374,9 @@
"license": "Apache-2.0"
},
"node_modules/@actions/artifact/node_modules/brace-expansion": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
"integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz",
"integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
@@ -2843,9 +2843,9 @@
}
},
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
"version": "5.0.9",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz",
"integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -3864,9 +3864,9 @@
"license": "MIT"
},
"node_modules/brace-expansion": {
"version": "1.1.18",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
"integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
"version": "1.1.16",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz",
"integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==",
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0",
@@ -5115,16 +5115,16 @@
}
},
"node_modules/eslint-plugin-import-x/node_modules/brace-expansion": {
"version": "5.0.9",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^4.0.2"
},
"engines": {
"node": "20 || >=22"
"node": "18 || 20 || >=22"
}
},
"node_modules/eslint-plugin-import-x/node_modules/minimatch": {
@@ -6111,15 +6111,15 @@
}
},
"node_modules/glob/node_modules/brace-expansion": {
"version": "5.0.9",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
"license": "MIT",
"dependencies": {
"balanced-match": "^4.0.2"
},
"engines": {
"node": "20 || >=22"
"node": "18 || 20 || >=22"
}
},
"node_modules/glob/node_modules/minimatch": {
@@ -6138,9 +6138,9 @@
}
},
"node_modules/globals": {
"version": "17.8.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-17.8.0.tgz",
"integrity": "sha512-Zz/LMDZScFmkakeL2cTHzf+PbWKdpU3uclqkZT7TjDG58j5WPt0PpA+n9uPI24fZtlw07q0OtEi84K+umsRzqQ==",
"version": "17.7.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-17.7.0.tgz",
"integrity": "sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==",
"dev": true,
"license": "MIT",
"engines": {
@@ -6981,9 +6981,9 @@
}
},
"node_modules/js-yaml": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
"integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
"integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
"funding": [
{
"type": "github",
@@ -8090,15 +8090,15 @@
}
},
"node_modules/readdir-glob/node_modules/brace-expansion": {
"version": "5.0.9",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
"license": "MIT",
"dependencies": {
"balanced-match": "^4.0.2"
},
"engines": {
"node": "20 || >=22"
"node": "18 || 20 || >=22"
}
},
"node_modules/readdir-glob/node_modules/minimatch": {
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "4.37.7",
"version": "4.37.5",
"private": true,
"description": "CodeQL action",
"scripts": {
@@ -39,7 +39,7 @@
"follow-redirects": "^1.16.0",
"get-folder-size": "^5.0.0",
"https-proxy-agent": "^7.0.6",
"js-yaml": "^5.2.2",
"js-yaml": "^5.2.1",
"jsonschema": "1.5.0",
"long": "^5.3.2",
"node-forge": "^1.4.0",
@@ -69,7 +69,7 @@
"eslint-plugin-jsdoc": "^62.9.0",
"eslint-plugin-no-async-foreach": "^0.1.1",
"glob": "^13.0.6",
"globals": "^17.8.0",
"globals": "^17.7.0",
"nock": "^14.0.16",
"sinon": "^22.1.0",
"typescript": "^6.0.3",
+68
View File
@@ -0,0 +1,68 @@
/**
* Tests for `check-cs-config.ts`.
*/
import * as assert from "node:assert/strict";
import { describe, it } from "node:test";
import type { UserConfig } from "../src/config/db-config";
import { checkConfiguration } from "./check-cs-config";
describe("checkConfiguration", async () => {
await it("passes when actual and expected configs match", () => {
const actual: UserConfig = { name: "test-config", paths: ["src"] };
const expected = JSON.stringify(actual);
assert.doesNotThrow(() => checkConfiguration(actual, expected));
});
await it("passes when queries arrays match after sorting", () => {
const actual: UserConfig = { paths: ["b", "a", "c"] };
const expected = JSON.stringify(actual);
assert.doesNotThrow(() => checkConfiguration(actual, expected));
});
await it("throws when actual config does not match expected", () => {
const actual: UserConfig = { name: "actual-name" };
const expected = JSON.stringify({
name: "expected-name",
} satisfies UserConfig);
assert.throws(() => checkConfiguration(actual, expected), {
message: /Expected configuration does not match actual configuration/,
});
});
await it("throws when expected contents are empty", () => {
assert.throws(() => checkConfiguration({}, ""), {
message: /No expected configuration provided/,
});
});
await it("throws when expected contents are only whitespace", () => {
assert.throws(() => checkConfiguration({}, " "), {
message: /No expected configuration provided/,
});
});
await it("passes with complex config", () => {
const actual: UserConfig = {
name: "complex",
"disable-default-queries": true,
paths: ["src", "lib"],
"paths-ignore": ["test"],
"threat-models": ["remote"],
};
const expected = JSON.stringify(actual);
assert.doesNotThrow(() => checkConfiguration(actual, expected));
});
await it("trims whitespace from expected contents before parsing", () => {
const actual: UserConfig = { name: "trimmed" };
const expected = ` ${JSON.stringify(actual)} `;
assert.doesNotThrow(() => checkConfiguration(actual, expected));
});
await it("passes when both configs are empty objects", () => {
assert.doesNotThrow(() => checkConfiguration({}, "{}"));
});
});
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env npx tsx
/**
* Checks the code scanning configuration file generated by the
* action to ensure it contains the expected contents
*/
import * as assert from "node:assert";
import * as fs from "node:fs";
import { parseArgs } from "node:util";
import * as core from "@actions/core";
import * as yaml from "yaml";
import type { UserConfig } from "../src/config/db-config";
import { getErrorMessage } from "./util";
function sortConfigArrays(config: UserConfig) {
for (const key of Object.keys(config)) {
const value = config[key];
if (key === "queries" && Array.isArray(value)) {
config[key] = value.sort();
}
}
return config;
}
function loadActualConfig(configPath: string) {
if (!fs.existsSync(configPath)) {
throw new Error("No configuration file found");
} else {
const rawActualConfig = fs.readFileSync(configPath, "utf8");
core.startGroup("Actual generated user config");
core.info(rawActualConfig);
core.endGroup();
return yaml.parse(rawActualConfig) as UserConfig;
}
}
export function checkConfiguration(
actualConfig: UserConfig,
expectedContents: string,
) {
const rawExpectedConfig = expectedContents.trim();
if (!rawExpectedConfig) {
throw new Error("No expected configuration provided");
}
const expectedConfig = JSON.parse(rawExpectedConfig) as UserConfig;
core.startGroup("Expected generated user config");
core.info(yaml.stringify(expectedConfig));
core.endGroup();
assert.deepStrictEqual(
sortConfigArrays(actualConfig),
sortConfigArrays(expectedConfig),
"Expected configuration does not match actual configuration",
);
}
function main() {
const { values } = parseArgs({
options: {
// The path of the configuration file to check.
file: {
type: "string",
},
// The expected contents of the file.
"expected-contents": {
type: "string",
},
},
strict: true,
});
if (values.file === undefined) {
throw new Error("The '--file' input is required.");
}
if (values["expected-contents"] === undefined) {
throw new Error("The '--expected-contents' input is required.");
}
const actualConfig = loadActualConfig(values.file);
try {
checkConfiguration(actualConfig, values["expected-contents"]);
} catch (err) {
core.error(getErrorMessage(err));
return -1;
}
return 0;
}
if (require.main === module) {
process.exit(main());
}
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env npx tsx
import * as fs from "node:fs";
import * as path from "node:path";
import * as core from "@actions/core";
import { runCommand, runGit } from "./command";
import { LIB_ROOT, PR_CHECKS_DIR, REPO_ROOT } from "./config";
import { getErrorMessage } from "./util";
function main() {
// Sanity check that repo is clean to start with
try {
runGit(["diff", "--exit-code"], { allowNonZeroExitCode: false });
console.info("Repository is clean.");
} catch (err) {
// If we get a fail here then this workflow needs attention...
console.error(getErrorMessage(err));
console.error("Failed: Repo should be clean before testing!");
return -1;
}
// Wipe the lib directory in case there are extra unnecessary files in there
console.info(`Removing ${LIB_ROOT}...`);
fs.rmSync(LIB_ROOT, { recursive: true, force: true });
// Generate the JavaScript files
runCommand("npm", ["run", "build"], { execOptions: { shell: true } });
// Check that repo is still clean
try {
runGit(["diff", "--exit-code"], { allowNonZeroExitCode: false });
console.info("Repository is clean.");
} catch (err) {
// If we get a fail here then the PR needs attention
console.error(getErrorMessage(err));
console.error("Failed: JavaScript files are not up to date.");
console.error("Run 'rm -rf lib && npm run build' to update.");
const diffFile = path.join(
process.env["RUNNER_TEMP"] ?? PR_CHECKS_DIR,
"js.diff",
);
runCommand("git", ["status"]);
runCommand("git", ["diff", `--output=${diffFile}`], {
execOptions: { cwd: REPO_ROOT },
});
core.summary.addHeading("Transpiled JS diff", 3);
core.summary.addCodeBlock(fs.readFileSync(diffFile, "utf-8"), "diff");
fs.rmSync(diffFile);
// Reset bundled files to allow other checks to test for changes
runCommand("git", ["checkout", "lib"]);
return 1;
}
console.info("Success: JavaScript files are up to date");
return 0;
}
if (require.main === module) {
process.exit(main());
}
+159
View File
@@ -0,0 +1,159 @@
/**
* Tests for `check-sarif.ts`.
*/
import * as assert from "node:assert/strict";
import { describe, it } from "node:test";
import type { Log } from "sarif";
import { checkSarif } from "./check-sarif";
/** Builds a minimal SARIF Log with the given rule IDs spread across extensions. */
function buildSarifLog(ruleIds: string[]): Log {
return {
version: "2.1.0",
$schema:
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
runs: [
{
tool: {
driver: { name: "CodeQL" },
extensions: [
{
name: "test-pack",
rules: ruleIds.map((id) => ({ id })),
},
],
},
results: [],
},
],
};
}
describe("checkSarif", async () => {
await it("returns 0 when all expected queries ran and no unexpected queries ran", () => {
const sarif = buildSarifLog(["js/sql-injection", "js/xss"]);
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/sql-injection, js/xss",
queriesNotRun: "js/hardcoded-credentials",
});
assert.equal(exitCode, 0);
});
await it("returns -2 when an expected query did not run", () => {
const sarif = buildSarifLog(["js/sql-injection"]);
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/sql-injection, js/xss",
queriesNotRun: "",
});
assert.equal(exitCode, -2);
});
await it("returns -2 when an unexpected query ran", () => {
const sarif = buildSarifLog(["js/sql-injection", "js/xss"]);
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/sql-injection",
queriesNotRun: "js/xss",
});
assert.equal(exitCode, -2);
});
await it("handles empty queries-run and queries-not-run inputs", () => {
const sarif = buildSarifLog(["js/sql-injection"]);
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "",
queriesNotRun: "",
});
assert.equal(exitCode, 0);
});
await it("handles multiple extensions with rules", () => {
const sarif: Log = {
version: "2.1.0",
$schema:
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
runs: [
{
tool: {
driver: { name: "CodeQL" },
extensions: [
{
name: "pack-a",
rules: [{ id: "js/sql-injection" }],
},
{
name: "pack-b",
rules: [{ id: "js/xss" }],
},
],
},
results: [],
},
],
};
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/sql-injection, js/xss",
queriesNotRun: "",
});
assert.equal(exitCode, 0);
});
await it("handles extensions with no rules", () => {
const sarif: Log = {
version: "2.1.0",
$schema:
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
runs: [
{
tool: {
driver: { name: "CodeQL" },
extensions: [
{ name: "empty-pack" },
{
name: "pack-with-rules",
rules: [{ id: "js/xss" }],
},
],
},
results: [],
},
],
};
const exitCode = checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/xss",
queriesNotRun: "js/sql-injection",
});
assert.equal(exitCode, 0);
});
await it("throws when tool extensions are undefined", () => {
const sarif: Log = {
version: "2.1.0",
$schema:
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json",
runs: [
{
tool: { driver: { name: "CodeQL" } },
results: [],
},
],
};
assert.throws(
() =>
checkSarif(sarif, {
sarifFile: "test.sarif",
queriesRun: "js/xss",
queriesNotRun: "",
}),
{ message: /Couldn't find tool extensions/ },
);
});
});
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env npx tsx
/** Checks a SARIF file to see if certain queries were run and others were not run. */
import * as fs from "node:fs";
import { parseArgs } from "node:util";
import * as core from "@actions/core";
import type { ReportingDescriptor, Log } from "sarif";
import { getErrorMessage } from "./util";
type Options = { sarifFile: string; queriesRun: string; queriesNotRun: string };
function getOptions(): Options {
const { values } = parseArgs({
options: {
// The path of the SARIF file to check.
"sarif-file": {
type: "string",
},
// The query ids to check are present.
"queries-run": {
type: "string",
},
// The query ids to check are absent.
"queries-not-run": {
type: "string",
},
},
strict: true,
});
if (values["sarif-file"] === undefined) {
throw new Error("The '--sarif-file' input is required.");
}
if (values["queries-run"] === undefined) {
throw new Error("The '--queries-run' input is required.");
}
if (values["queries-not-run"] === undefined) {
throw new Error("The '--queries-not-run' input is required.");
}
return {
sarifFile: values["sarif-file"],
queriesRun: values["queries-run"],
queriesNotRun: values["queries-not-run"],
};
}
function parseQueryIdsInput(queriesRun: string): string[] {
return queriesRun
.split(",")
.map((q) => q.trim())
.filter((q) => q.length > 0);
}
export function checkSarif(sarif: Log, options: Options) {
if (sarif.runs[0].tool.extensions === undefined) {
throw new Error(`Couldn't find tool extensions in the SARIF file.`);
}
let exitCode = 0;
// Extract the rule ids from the SARIF file.
const rules: ReportingDescriptor[] = sarif.runs[0].tool.extensions.flatMap(
(ext) => ext.rules || [],
);
const ruleIds: string[] = rules.map((rule) => rule.id);
// Check that all the expected queries ran
const expectedQueriesRun = parseQueryIdsInput(options.queriesRun);
const queriesThatShouldHaveRunButDidNot = expectedQueriesRun.filter(
(queryId) => !ruleIds.includes(queryId),
);
if (queriesThatShouldHaveRunButDidNot.length > 0) {
core.error(
`The following queries were expected to run but did not: ${queriesThatShouldHaveRunButDidNot.join(", ")}`,
);
exitCode = -2;
}
// Check that all the unexpected queries did not run
const expectedQueriesNotRun = parseQueryIdsInput(options.queriesNotRun);
const queriesThatShouldNotHaveRunButDid = expectedQueriesNotRun.filter(
(queryId) => ruleIds.includes(queryId),
);
if (queriesThatShouldNotHaveRunButDid.length > 0) {
core.error(
`The following queries were NOT expected to have run but did: ${queriesThatShouldNotHaveRunButDid.join(", ")}`,
);
exitCode = -2;
}
core.startGroup("All queries that ran");
for (const rule of rules) {
core.info(`${rule.id}: ${rule.properties?.name || rule.name}`);
}
core.endGroup();
core.startGroup("Full SARIF");
core.info(JSON.stringify(sarif, null, 2));
core.endGroup();
return exitCode;
}
function main() {
try {
const options = getOptions();
const sarif: Log = JSON.parse(fs.readFileSync(options.sarifFile, "utf8"));
return checkSarif(sarif, options);
} catch (err) {
core.error(`Failed to check SARIF file: ${getErrorMessage(err)}`);
return -1;
}
}
if (require.main === module) {
process.exit(main());
}
+81
View File
@@ -0,0 +1,81 @@
import { execFileSync, ExecFileSyncOptions } from "node:child_process";
import { DryRunOption, REPO_ROOT } from "./config";
/** Options for {@link runCommand}. */
export interface RunCommandOptions extends DryRunOption {
/** Options for `execFileSync`. */
execOptions?: ExecFileSyncOptions;
}
/**
* Runs a command, streaming output to the console by default.
*
* @param command The name of the command to run.
* @param args The arguments for the command.
* @throws When the process exits with a non-zero exit code.
* @param options How to run the command.
*/
export function runCommand(
command: string,
args: string[],
options?: RunCommandOptions,
) {
if (!options?.dryRun) {
console.log(`Running \`${command} ${args.join(" ")}\`.`);
return execFileSync(command, args, {
stdio: "inherit",
cwd: REPO_ROOT,
...options?.execOptions,
});
} else {
console.info(
`[DRY RUN] Would have executed '${command} ${args.join(" ")}'`,
);
return "";
}
}
/** Options for {@link runGit}. */
export interface RunGitOptions extends DryRunOption {
/** When true, non-zero exit codes will not throw. */
allowNonZeroExitCode?: boolean;
}
/**
* Runs `git` with the given `args` and returns the stdout.
*
* @param args - Arguments to pass to `git`.
* @param options - Optional settings.
* @throws If `git` does not exit successfully, unless
* `options.allowNonZeroExitCode` is `true`.
* @returns The trimmed stdout output.
*/
export function runGit(args: string[], options?: RunGitOptions): string {
const execOptions: ExecFileSyncOptions = {
encoding: "utf8",
stdio: ["pipe", "pipe", "pipe"],
};
try {
const result = runCommand("git", args, {
dryRun: options?.dryRun,
execOptions,
}) as string;
return result.trimEnd();
} catch (error: unknown) {
if (options?.allowNonZeroExitCode) {
// execFileSync throws an object with `stdout` when the process exits
// with a non-zero code.
const execError = error as { stdout?: Buffer | string };
if (typeof execError.stdout === "string") {
return execError.stdout.trimEnd();
}
if (Buffer.isBuffer(execError.stdout)) {
return execError.stdout.toString("utf8").trimEnd();
}
return "";
}
throw error;
}
}
+7 -1
View File
@@ -22,7 +22,13 @@ export const CHANGELOG_FILE = path.join(REPO_ROOT, "CHANGELOG.md");
export const BUNDLE_METADATA_FILE = path.join(REPO_ROOT, "meta.json");
/** The `src` directory. */
const SOURCE_ROOT = path.join(REPO_ROOT, "src");
export const SOURCE_ROOT = path.join(REPO_ROOT, "src");
/** The `src` directory. */
export const LIB_ROOT = path.join(REPO_ROOT, "lib");
/** The path to `defaults.json`. */
export const DEFAULTS_FILE = path.join(SOURCE_ROOT, "defaults.json");
/** The path to the built-in languages file. */
export const BUILTIN_LANGUAGES_FILE = path.join(
+2 -2
View File
@@ -253,8 +253,8 @@ const languageSetups: LanguageSetups = {
name: "Install Java",
uses: pinnedUses(
"actions/setup-java",
"b6effb05e454b25005698d916606bdc6ffcbf961",
"v5.7.0",
"03ad4de0992f5dab5e18fcb136590ce7c4a0ac95",
"v5.6.0",
),
with: {
"java-version": `\${{ inputs.java-version || '${defaultLanguageVersions.java}' }}`,
+58
View File
@@ -0,0 +1,58 @@
/*
* Tests for the update-bundle.ts script.
*/
import * as assert from "node:assert/strict";
import { describe, it } from "node:test";
import { Defaults, getNewDefaults } from "./update-bundle";
const testDefaults: Defaults = {
bundleVersion: "codeql-bundle-v2.26.2",
cliVersion: "2.26.2",
priorBundleVersion: "codeql-bundle-v2.26.1",
priorCliVersion: "2.26.1",
};
describe("getNewDefaults", async () => {
await it("throws if there is no cli-version-*.txt asset", async () => {
assert.throws(
() => getNewDefaults({ tag_name: "foo", assets: [] }, testDefaults),
{ message: "Failed to find the CodeQL CLI version for release foo." },
);
});
await it("throws if there are multiple cli-version-*.txt assets", async () => {
assert.throws(
() =>
getNewDefaults(
{
tag_name: "foo",
assets: [
{ name: "cli-version-foo.txt" },
{ name: "cli-version-bar.txt" },
],
},
testDefaults,
),
{ message: "Release foo has multiple CLI version marker files." },
);
});
await it("finds the new bundle info", async () => {
const newDefaults = getNewDefaults(
{
tag_name: "foo",
assets: [{ name: "cli-version-1.2.3.txt" }],
},
testDefaults,
);
assert.deepEqual(newDefaults, {
bundleVersion: "foo",
cliVersion: "1.2.3",
priorBundleVersion: testDefaults.bundleVersion,
priorCliVersion: testDefaults.cliVersion,
} satisfies Defaults);
});
});
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env npx tsx
/** Updates 'src/defaults.json' to point to a new CodeQL bundle release. */
import * as fs from "fs";
import * as github from "@actions/github";
import * as defaults from "../src/defaults.json";
import { DEFAULTS_FILE } from "./config";
interface BundleInfo {
bundleVersion: string;
cliVersion: string;
}
export type Defaults = typeof defaults;
interface Release {
tag_name: string;
assets: Array<{
name: string;
}>;
}
function getCodeQLCliVersionForRelease(release: Release): string {
// We do not currently tag CodeQL bundles based on the CLI version they contain.
// Instead, we use a marker file `cli-version-<version>.txt` to record the CLI version.
// This marker file is uploaded as a release asset for all new CodeQL bundles.
const cliVersionsFromMarkerFiles = release.assets
.map((asset) => asset.name.match(/cli-version-(.*)\.txt/)?.[1])
.filter((v) => v)
.map((v) => v as string);
if (cliVersionsFromMarkerFiles.length > 1) {
throw new Error(
`Release ${release.tag_name} has multiple CLI version marker files.`,
);
} else if (cliVersionsFromMarkerFiles.length === 0) {
throw new Error(
`Failed to find the CodeQL CLI version for release ${release.tag_name}.`,
);
}
return cliVersionsFromMarkerFiles[0];
}
function getBundleInfoFromRelease(release: Release): BundleInfo {
return {
bundleVersion: release.tag_name,
cliVersion: getCodeQLCliVersionForRelease(release),
};
}
export function getNewDefaults(
release: Release,
currentDefaults: Defaults,
): Defaults {
console.log(
"Updating default bundle as a result of the following release: " +
`${JSON.stringify(release)}.`,
);
const bundleInfo = getBundleInfoFromRelease(release);
return {
bundleVersion: bundleInfo.bundleVersion,
cliVersion: bundleInfo.cliVersion,
priorBundleVersion: currentDefaults.bundleVersion,
priorCliVersion: currentDefaults.cliVersion,
};
}
function main() {
const release: Release = github.context.payload.release;
if (release === undefined) {
console.error(`Release payload is undefined.`);
return -1;
}
const previousDefaults = defaults;
const newDefaults = getNewDefaults(release, previousDefaults);
// Update the source file in the repository. Calling workflows should subsequently rebuild
// the Action to update `lib/defaults.json`.
fs.writeFileSync(DEFAULTS_FILE, `${JSON.stringify(newDefaults, null, 2)}\n`);
return 0;
}
if (require.main === module) {
process.exit(main());
}
+2 -80
View File
@@ -18,12 +18,12 @@
* [--dry-run]
*/
import { execFileSync, type ExecFileSyncOptions } from "node:child_process";
import { execFileSync } from "node:child_process";
import { parseArgs } from "node:util";
import { type ApiClient, getApiClient } from "./api-client";
import * as changelog from "./changelog";
import { DryRunOption, REPO_ROOT } from "./config";
import { runCommand, runGit } from "./command";
import {
getCurrentVersion,
replaceVersionInPackageJson,
@@ -65,84 +65,6 @@ export function getGitHubToken(): string {
throw new Error("Missing GitHub token. Set GITHUB_TOKEN or GH_TOKEN.");
}
/** Options for {@link runCommand}. */
export interface RunCommandOptions extends DryRunOption {
/** Options for `execFileSync`. */
execOptions?: ExecFileSyncOptions;
}
/**
* Runs a command, streaming output to the console by default.
*
* @param command The name of the command to run.
* @param args The arguments for the command.
* @throws When the process exits with a non-zero exit code.
* @param options How to run the command.
*/
export function runCommand(
command: string,
args: string[],
options?: RunCommandOptions,
) {
if (!options?.dryRun) {
console.log(`Running \`${command} ${args.join(" ")}\`.`);
return execFileSync(command, args, {
stdio: "inherit",
cwd: REPO_ROOT,
...options?.execOptions,
});
} else {
console.info(
`[DRY RUN] Would have executed '${command} ${args.join(" ")}'`,
);
return "";
}
}
/** Options for {@link runGit}. */
export interface RunGitOptions extends DryRunOption {
/** When true, non-zero exit codes will not throw. */
allowNonZeroExitCode?: boolean;
}
/**
* Runs `git` with the given `args` and returns the stdout.
*
* @param args - Arguments to pass to `git`.
* @param options - Optional settings.
* @throws If `git` does not exit successfully, unless
* `options.allowNonZeroExitCode` is `true`.
* @returns The trimmed stdout output.
*/
export function runGit(args: string[], options?: RunGitOptions): string {
const execOptions: ExecFileSyncOptions = {
encoding: "utf8",
stdio: ["pipe", "pipe", "pipe"],
};
try {
const result = runCommand("git", args, {
dryRun: options?.dryRun,
execOptions,
}) as string;
return result.trimEnd();
} catch (error: unknown) {
if (options?.allowNonZeroExitCode) {
// execFileSync throws an object with `stdout` when the process exits
// with a non-zero code.
const execError = error as { stdout?: Buffer | string };
if (typeof execError.stdout === "string") {
return execError.stdout.trimEnd();
}
if (Buffer.isBuffer(execError.stdout)) {
return execError.stdout.toString("utf8").trimEnd();
}
return "";
}
throw error;
}
}
/** Returns true if the given branch exists on the origin remote. */
export function branchExistsOnRemote(branchName: string): boolean {
const result = runGit(["ls-remote", "--heads", ORIGIN, branchName]);
+24 -98
View File
@@ -1295,12 +1295,33 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is above the default limit",
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 5_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisResourceChecksV2,
],
isDefaultBranch: true,
diskUsage: {
@@ -1315,7 +1336,7 @@ checkOverlayEnablementMacro.serial(
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below the default limit",
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
{
languages: [BuiltInLanguage.javascript],
features: [
@@ -1324,102 +1345,7 @@ checkOverlayEnablementMacro.serial(
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 10_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
// Check that each feature flag lowers the limit to the threshold that its name
// declares. Both sides of the boundary are needed to pin the threshold down: a
// mapping to a lower value would still pass the case at the limit, and one to a
// higher value would still fail the case below it.
for (const [feature, thresholdGb] of [
[Feature.OverlayAnalysisMinDisk8Gb, 8],
[Feature.OverlayAnalysisMinDisk9Gb, 9],
[Feature.OverlayAnalysisMinDisk10Gb, 10],
[Feature.OverlayAnalysisMinDisk11Gb, 11],
[Feature.OverlayAnalysisMinDisk12Gb, 12],
[Feature.OverlayAnalysisMinDisk13Gb, 13],
] as Array<[Feature, number]>) {
const features = [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
feature,
];
checkOverlayEnablementMacro.serial(
`Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
`No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`,
{
languages: [BuiltInLanguage.javascript],
features,
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
},
);
}
checkOverlayEnablementMacro.serial(
"Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 9_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
{
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
useOverlayDatabaseCaching: true,
},
);
checkOverlayEnablementMacro.serial(
"No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag",
{
languages: [BuiltInLanguage.javascript],
features: [
Feature.OverlayAnalysis,
Feature.OverlayAnalysisCodeScanningJavascript,
Feature.OverlayAnalysisMinDisk9Gb,
Feature.OverlayAnalysisMinDisk12Gb,
],
isDefaultBranch: true,
diskUsage: {
numAvailableBytes: 8_500_000_000,
numAvailableBytes: 15_000_000_000,
numTotalBytes: 100_000_000_000,
},
},
+24 -49
View File
@@ -48,7 +48,7 @@ import {
import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils";
import { EnvVar } from "./environment";
import * as errorMessages from "./error-messages";
import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags";
import { Feature, FeatureEnablement } from "./feature-flags";
import {
RepositoryProperties,
RepositoryPropertyName,
@@ -101,28 +101,19 @@ export { type Config } from "./config/action-config";
* whether to perform overlay analysis, then the action will not perform overlay
* analysis unless overlay analysis has been explicitly enabled via environment
* variable.
*
* This threshold can be lowered by the feature flags in
* `OVERLAY_MINIMUM_DISK_SPACE_FEATURES`.
*/
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
/**
* Feature flags that lower the minimum available disk space required to perform
* overlay analysis, paired with the threshold (in MB) that each one enables.
*
* If several of these are enabled, the lowest threshold takes effect.
* The v2 minimum available disk space (in MB) required to perform overlay
* analysis. This is a lower threshold than the v1 limit, allowing overlay
* analysis to run on runners with less available disk space.
*/
const OVERLAY_MINIMUM_DISK_SPACE_FEATURES: ReadonlyArray<
[FeatureWithoutCLI, number]
> = [
[Feature.OverlayAnalysisMinDisk8Gb, 8000],
[Feature.OverlayAnalysisMinDisk9Gb, 9000],
[Feature.OverlayAnalysisMinDisk10Gb, 10000],
[Feature.OverlayAnalysisMinDisk11Gb, 11000],
[Feature.OverlayAnalysisMinDisk12Gb, 12000],
[Feature.OverlayAnalysisMinDisk13Gb, 13000],
];
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000;
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES =
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000;
/**
* The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If
@@ -597,42 +588,24 @@ async function checkOverlayAnalysisFeatureEnabled(
return new Success(undefined);
}
/**
* Returns the minimum available disk space (in MB) required to perform overlay
* analysis, which is the lowest threshold enabled by a feature flag, or the
* default threshold if no such feature flag is enabled.
*/
async function getMinimumDiskSpaceMb(
features: FeatureEnablement,
): Promise<number> {
let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB;
for (const [feature, thresholdMb] of OVERLAY_MINIMUM_DISK_SPACE_FEATURES) {
if (await features.getValue(feature)) {
minimumMb = Math.min(minimumMb, thresholdMb);
}
}
return minimumMb;
}
/** Checks if the runner has enough disk space for overlay analysis. */
function runnerHasSufficientDiskSpace(
diskUsage: DiskUsage,
logger: Logger,
minimumDiskSpaceMb: number,
useV2ResourceChecks: boolean,
): boolean {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) {
const minimumDiskSpaceBytes = useV2ResourceChecks
? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES
: OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) {
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
logger.info(
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
);
return false;
}
logger.debug(
`Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` +
`minimum of ${minimumDiskSpaceMb} MB.`,
);
return true;
}
@@ -664,7 +637,7 @@ async function runnerHasSufficientMemory(
}
logger.debug(
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
);
return true;
}
@@ -675,13 +648,12 @@ async function runnerHasSufficientMemory(
*/
async function checkRunnerResources(
codeql: CodeQL,
features: FeatureEnablement,
diskUsage: DiskUsage,
ramInput: string | undefined,
logger: Logger,
useV2ResourceChecks: boolean,
): Promise<Result<void, OverlayDisabledReason>> {
const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features);
if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) {
if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) {
return new Failure(OverlayDisabledReason.InsufficientDiskSpace);
}
if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) {
@@ -780,6 +752,9 @@ export async function checkOverlayEnablement(
Feature.OverlayAnalysisSkipResourceChecks,
codeql,
));
const useV2ResourceChecks = await features.getValue(
Feature.OverlayAnalysisResourceChecksV2,
);
const checkOverlayStatus = await features.getValue(
Feature.OverlayAnalysisStatusCheck,
);
@@ -795,10 +770,10 @@ export async function checkOverlayEnablement(
performResourceChecks && diskUsage !== undefined
? await checkRunnerResources(
codeql,
features,
diskUsage,
ramInput,
logger,
useV2ResourceChecks,
)
: new Success<void>(undefined);
if (resourceResult.isFailure()) {
+1 -1
View File
@@ -16,7 +16,7 @@ export interface RemoteFileAddress {
}
/** The default file path to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-config.yml";
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-action.yaml";
/** The default ref to use in configuration file shorthands. */
export const DEFAULT_CONFIG_FILE_REF = "main";
+7 -39
View File
@@ -121,19 +121,12 @@ export enum Feature {
* `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
*/
OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
/**
* Feature flags that lower the amount of available disk space that the overlay hardware check
* requires. The lowest threshold that is enabled takes effect; if none are enabled, the default
* threshold applies. These flags have no effect if `OverlayAnalysisSkipResourceChecks` is
* enabled.
*/
OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb",
OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb",
OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb",
OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb",
OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb",
OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb",
OverlayAnalysisPython = "overlay_analysis_python",
/**
* Controls whether lower disk space requirements are used for overlay hardware checks.
* Has no effect if `OverlayAnalysisSkipResourceChecks` is enabled.
*/
OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2",
OverlayAnalysisRuby = "overlay_analysis_ruby",
/** Controls whether hardware checks are skipped for overlay analysis. */
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
@@ -361,34 +354,9 @@ export const featureConfig = {
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk8Gb]: {
[Feature.OverlayAnalysisResourceChecksV2]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk9Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk10Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk11Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk12Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisMinDisk13Gb]: {
defaultValue: false,
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB",
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",
minimumVersion: undefined,
},
[Feature.OverlayAnalysisStatusCheck]: {
+5 -27
View File
@@ -35,11 +35,6 @@ export function isNumber(value: unknown): value is number {
return typeof value === "number";
}
/** Asserts that `value` is a boolean. */
export function isBoolean(value: unknown): value is boolean {
return typeof value === "boolean";
}
/** Asserts that `value` is either a string or undefined. */
export function isStringOrUndefined(
value: unknown,
@@ -67,11 +62,14 @@ function defaultCheck(
return (arg) => ({ unknownKeys: [], invalidKeys: [], valid: validate(arg) });
}
function makeValidator<T>(validate: (arg: unknown) => arg is T) {
function makeValidator<T>(
validate: (arg: unknown) => arg is T,
required: boolean = true,
) {
return {
validate,
check: defaultCheck(validate),
required: true,
required,
} as const satisfies Validator<T>;
}
@@ -84,9 +82,6 @@ export const string = makeValidator(isString);
/** A validator for number fields in schemas. */
export const number = makeValidator(isNumber);
/** A validator for boolean fields in schemas. */
export const boolean = makeValidator(isBoolean);
/** A validator for arrays. */
export function array<T>(validator: Validator<T>) {
const validate = (val: unknown) => {
@@ -226,23 +221,6 @@ export function validateSchema<
return result.valid;
}
/**
* Validates that `arr` is an array whose elements satisfy at least `elementSchema`.
* Additional keys are accepted in each element.
*
* @param elementSchema The schema to validate the elements against.
* @param arr The array to validate.
* @returns Asserts that `arr` has elements of `schema`'s type if validation is successful.
*/
export function validateArray<
S extends Schema,
T extends UnvalidatedArray = Array<FromSchema<S>>,
>(elementSchema: S, arr: UnvalidatedArray): arr is T {
const elementValidator = object(elementSchema);
return array(elementValidator).validate(arr);
}
export interface CheckSchemaOptions {
/** Whether to stop validation after the first error. */
failFast?: boolean;
+7 -1
View File
@@ -83,6 +83,12 @@ export class StartProxyError extends Error {
}
}
interface StartProxyStatus extends StatusReportBase {
// A comma-separated list of registry types which are configured for CodeQL.
// This only includes registry types we support, not all that are configured.
registry_types: string;
}
/**
* Sends a status report for the `start-proxy` action indicating a successful outcome.
*
@@ -106,7 +112,7 @@ export async function sendSuccessStatusReport(
logger,
);
if (statusReportBase !== undefined) {
const statusReport: StatusReportBase = {
const statusReport: StartProxyStatus = {
...statusReportBase,
registry_types: registry_types.join(","),
};
+6 -12
View File
@@ -254,19 +254,13 @@ export function credentialToStr(credential: Credential): string {
return result;
}
/** The schema for `RegistryBase` objects. */
export const registryBaseSchema = {
/** The type of the package registry. */
type: json.string,
/** Whether the registry replaces the base registry for the ecosystem. */
"replaces-base": json.optional(json.boolean),
} as const satisfies json.Schema;
/** Information about a registry, other than its address. */
export type RegistryBase = json.FromSchema<typeof registryBaseSchema>;
/** A package registry is identified by its type and address. */
export type Registry = RegistryBase & Address;
export type Registry = {
/** The type of the package registry. */
type: string;
/** Whether the registry replaces the base registry for the ecosystem. */
"replaces-base"?: boolean;
} & Address;
// If a registry has an `url`, then that takes precedence over the `host` which may or may
// not be defined.
+1 -64
View File
@@ -4,17 +4,15 @@ import * as uuid from "uuid";
import * as actionsUtil from "./actions-util";
import { Config } from "./config-utils";
import { EnvVar, RegistryProxyVars } from "./environment";
import { EnvVar } from "./environment";
import { BuiltInLanguage } from "./languages";
import { getRunnerLogger } from "./logging";
import { ToolsSource } from "./setup-codeql";
import type { Registry } from "./start-proxy";
import {
ActionName,
createInitWithConfigStatusReport,
createStatusReportBase,
getActionsStatus,
getRegistryTypesFromEnv,
getJobUUID,
InitStatusReport,
InitWithConfigStatusReport,
@@ -24,69 +22,12 @@ import {
setupActionsVars,
createTestConfig,
makeMacro,
getTestEnv,
RecordingLogger,
callee,
} from "./testing-utils";
import { BuildMode, ConfigurationError, withTmpDir, wrapError } from "./util";
setupTests(test);
test("getRegistryTypesFromEnv - gets unique registry types from environment", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([
{ type: "git_source", url: "https://example.com" },
{ type: "git_source", url: "https://github.com" },
{ type: "docker_registry", url: "https://registry.example.com" },
] satisfies Array<Partial<Registry>>),
});
const result = getRegistryTypesFromEnv(logger, env);
t.deepEqual(result, ["git_source", "docker_registry"].sort().join(","));
});
test("getRegistryTypesFromEnv - returns undefined if the env var is not set", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({});
const result = getRegistryTypesFromEnv(logger, env);
t.is(result, undefined);
});
test("getRegistryTypesFromEnv - returns undefined if the env var is not valid JSON", async (t) => {
const logger = new RecordingLogger(true);
const env = getTestEnv({ [RegistryProxyVars.PROXY_URLS]: "[" });
const result = getRegistryTypesFromEnv(logger, env);
t.is(result, undefined);
});
test("getRegistryTypesFromEnv - returns undefined if the env var is unexpected JSON", async (t) => {
const logger = new RecordingLogger(true);
t.is(
getRegistryTypesFromEnv(
logger,
getTestEnv({
// Top-level object rather than an array of objects.
[RegistryProxyVars.PROXY_URLS]: JSON.stringify({ type: "git_source" }),
}),
),
undefined,
);
t.is(
getRegistryTypesFromEnv(
logger,
getTestEnv({
// Object has no "type" key.
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([{}]),
}),
),
undefined,
);
});
test("getJobUUID - generates valid UUIDs", async (t) => {
await callee(getJobUUID)
.withArgs()
@@ -133,9 +74,6 @@ function setupEnvironmentAndStub(tmpDir: string) {
process.env[EnvVar.ANALYSIS_KEY] = "analysis-key";
process.env["ImageVersion"] = "2023.05.19.1";
process.env[RegistryProxyVars.PROXY_URLS] = JSON.stringify([
{ type: "maven_repository" },
] satisfies Array<Partial<Registry>>);
const getRequiredInput = sinon.stub(actionsUtil, "getRequiredInput");
getRequiredInput.withArgs("matrix").resolves("input/matrix");
@@ -179,7 +117,6 @@ test.serial("createStatusReportBase", async (t) => {
t.is(typeof statusReport.job_run_uuid, "string");
t.is(statusReport.languages, "java,swift");
t.is(statusReport.ref, process.env["GITHUB_REF"]!);
t.is(statusReport.registry_types, "maven_repository");
t.is(statusReport.runner_available_disk_space_bytes, 100);
t.is(statusReport.runner_image_version, process.env["ImageVersion"]);
t.is(statusReport.runner_os, process.env["RUNNER_OS"]!);
+1 -54
View File
@@ -19,14 +19,12 @@ import type { ComputedInput, InputName } from "./config/inputs";
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
import type { DependencyCacheRestoreStatusReport } from "./dependency-caching";
import { DocUrl } from "./doc-url";
import { EnvVar, getEnv, ReadOnlyEnv, RegistryProxyVars } from "./environment";
import { EnvVar } from "./environment";
import { getRef } from "./git-utils";
import * as json from "./json";
import type { Logger } from "./logging";
import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
import { getRepositoryNwo } from "./repository";
import type { ToolsSource } from "./setup-codeql";
import { registryBaseSchema } from "./start-proxy/types";
import {
ConfigurationError,
getRequiredEnvParam,
@@ -187,12 +185,6 @@ export interface StatusReportBase {
ml_powered_javascript_queries?: string;
/** Ref that the workflow was triggered on. */
ref: string;
/**
* A comma-separated list of private registry types which are configured for CodeQL.
* This only includes registry types we support (as determined by the `start-proxy` action),
* not all that are configured.
*/
registry_types?: string;
/** Action runner hardware architecture (context runner.arch). */
runner_arch?: string;
/** Available disk space on the runner, in bytes. */
@@ -296,50 +288,6 @@ export interface EventReport {
started_at: string;
}
/**
* Attempts to retrieve a list of private registry types from the `CODEQL_PROXY_URLS` environment
* variable and returns it as a comma-separated string if successful. Returns `undefined` otherwise.
*/
export function getRegistryTypesFromEnv(
logger: Logger,
env: ReadOnlyEnv = getEnv(),
): string | undefined {
// Try to get the value of the environment variable.
const value = env.getOptional(RegistryProxyVars.PROXY_URLS);
if (value === undefined) {
return undefined;
}
// Try to parse the JSON we expect to find in it and return the comma-separated list of
// (unique) registry types.
try {
const data = JSON.parse(value) as unknown;
// Check that the parsed JSON meets our expectations.
if (!json.isArray(data)) {
logger.debug(
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array, but got '${typeof data}'.`,
);
return undefined;
}
if (!json.validateArray(registryBaseSchema, data)) {
logger.debug(
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array of registry objects, but got something else.`,
);
return undefined;
}
const types = new Set(data.map((r) => r.type));
return Array.from(types).sort().join(",");
} catch (err) {
logger.debug(
`Failed to parse '${RegistryProxyVars.PROXY_URLS}': ${getErrorMessage(err)}.`,
);
return undefined;
}
}
/**
* Compose a StatusReport.
*
@@ -402,7 +350,6 @@ export async function createStatusReportBase(
job_name: jobName,
job_run_uuid: jobRunUUID,
ref,
registry_types: getRegistryTypesFromEnv(logger),
runner_os: runnerOs,
started_at: workflowStartedAt,
status,
-33
View File
@@ -1,33 +0,0 @@
import * as path from "path";
import * as stream from "stream";
import test from "ava";
import { getRunnerLogger } from "./logging";
import { extractTarZst } from "./tar";
import { setupTests } from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
test("extractTarZst rejects if the input stream errors", async (t) => {
await withTmpDir(async (tmpDir) => {
const archive = new stream.PassThrough();
const promise = extractTarZst(
archive,
path.join(tmpDir, "dest"),
{ type: "gnu", version: "1.34" },
getRunnerLogger(true),
);
archive.destroy(
Object.assign(new Error("socket hang up"), {
code: "ECONNRESET",
}),
);
await t.throwsAsync(promise, {
message: /Error while downloading and extracting tar/,
});
});
});
+4 -9
View File
@@ -194,15 +194,10 @@ export async function extractTarZst(
});
if (tar instanceof stream.Readable) {
// Use `pipeline` rather than `pipe` so that an error on either stream is reported here
// rather than being emitted as an unhandled `error` event, and so that `tar`'s standard
// input is closed if the download fails partway through.
stream.pipeline(tar, tarProcess.stdin, (err) => {
if (err) {
reject(
new Error(`Error while downloading and extracting tar: ${err}`),
);
}
tar.pipe(tarProcess.stdin).on("error", (err) => {
reject(
new Error(`Error while downloading and extracting tar: ${err}`),
);
});
}
-37
View File
@@ -38,43 +38,6 @@ test.serial(
},
);
test.serial(
"downloadAndExtract falls back to downloading before extracting if streaming fails",
async (t) => {
await withTmpDir(async (tmpDir) => {
sinon.stub(process, "platform").value("linux");
const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst");
const destination = path.join(tmpDir, "codeql");
const downloadTool = sinon
.stub(toolcache, "downloadTool")
.resolves(archivePath);
const extract = sinon.stub(tar, "extract").resolves(destination);
const extractTarZst = sinon.stub(tar, "extractTarZst").resolves();
const request = nock("https://example.com")
.get("/codeql-bundle.tar.zst")
.replyWithError(
Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }),
);
const statusReport = await downloadAndExtract(
"https://example.com/codeql-bundle.tar.zst",
"zstd",
destination,
undefined,
{},
{ type: "gnu", version: "1.34" },
getRunnerLogger(true),
);
t.assert(Number.isInteger(statusReport.downloadDurationMs));
t.true(request.isDone());
t.false(extractTarZst.called);
t.true(downloadTool.calledOnce);
t.true(extract.calledOnce);
});
},
);
test.serial(
"downloadAndExtract omits the download duration when streaming extraction",
async (t) => {
+4 -24
View File
@@ -19,12 +19,6 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
*/
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
/**
* How long the streaming download of the CodeQL tools may stall for before we abort it. This
* applies both to establishing the connection and to gaps between chunks of the response body.
*/
const STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
/**
* The name of the tool cache directory for the CodeQL tools.
*/
@@ -143,8 +137,8 @@ async function downloadAndExtractZstdWithStreaming(
authorization ? { authorization } : {},
headers,
);
const response = await new Promise<IncomingMessage>((resolve, reject) => {
const request = https.get(
const response = await new Promise<IncomingMessage>((resolve) =>
https.get(
codeqlURL,
{
headers,
@@ -154,24 +148,10 @@ async function downloadAndExtractZstdWithStreaming(
agent,
} as unknown as RequestOptions,
(r) => resolve(r),
);
// Without this listener, connection failures such as `ECONNRESET` are emitted as unhandled
// `error` events, which terminate the process instead of letting us fall back to downloading
// the bundle before extracting it. This listener stays attached after the response arrives, so
// it also handles errors that occur while the response is being streamed.
request.on("error", reject);
request.setTimeout(STREAMING_STALL_TIMEOUT_MS, () => {
request.destroy(
new Error(
`No data received for ${formatDuration(STREAMING_STALL_TIMEOUT_MS)}.`,
),
);
});
});
),
);
if (response.statusCode !== 200) {
// Discard the response body so that the connection can be released.
response.resume();
throw new Error(
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
);