Compare commits

...

30 Commits

Author SHA1 Message Date
Angela P Wen 81644f35ff Add max line length of 120 to linter (#1524) 2023-02-07 14:09:33 +00:00
Henry Mercer 9ab6aa64a0 Merge pull request #1526 from github/mergeback/v2.2.2-to-main-39d8d7e7
Mergeback v2.2.2 refs/heads/releases/v2 into main
2023-02-06 20:23:48 +00:00
github-actions[bot] 256973e279 Update checked-in dependencies 2023-02-06 20:02:57 +00:00
github-actions[bot] 59b25b480f Update changelog and version after v2.2.2 2023-02-06 19:48:14 +00:00
Henry Mercer 39d8d7e78f Merge pull request #1525 from github/update-v2.2.2-927de483f
Merge main into releases/v2
2023-02-06 19:46:06 +00:00
Angela P Wen 39c954c513 Support security-experimental as a well-known suite (#1519) 2023-02-06 19:26:03 +00:00
github-actions[bot] 8af83634ca Update changelog for v2.2.2 2023-02-06 19:16:08 +00:00
Henry Mercer 927de483f0 Merge pull request #1523 from github/henrymercer/fix/cli-version-for-different-bundles
Fix toolcache behavior when downloading bundle from another repo
2023-02-06 19:05:45 +00:00
Henry Mercer e4c0a1b24d Merge branch 'main' into henrymercer/fix/cli-version-for-different-bundles 2023-02-06 18:24:11 +00:00
Henry Mercer d3962273b3 Merge pull request #1517 from github/henrymercer/fix/not-all-bundle-urls-contain-tag
Fix assumption that all CodeQL bundle URLs contain the tag name of the bundle
2023-02-06 18:20:21 +00:00
Henry Mercer c3cb270725 Merge pull request #1521 from MahmoudMabrok/patch-1
docs: add direct link to website
2023-02-06 16:34:01 +00:00
Henry Mercer 2b674f7ab9 Fix toolcache behavior when downloading bundle from another repo 2023-02-06 16:25:07 +00:00
Henry Mercer 6d47a7c8b1 Add regression test for bundle from different repo 2023-02-06 16:25:07 +00:00
Henry Mercer c6ff11c1c4 Add changelog note 2023-02-06 16:24:25 +00:00
Henry Mercer d3f2b2e6d2 Warn when multiple bundles for a single CLI are found in the toolcache 2023-02-06 12:28:33 +00:00
Henry Mercer d49282c3b5 Rename forceLatest to forceShippedTools 2023-02-06 11:57:48 +00:00
Mahmoud Mabrok Fouad c5c475188a docs: add direct link to website
To make it easy for users to go to website for more info.
2023-02-05 13:56:35 +02:00
Henry Mercer f140af5e28 Refactor setting up CodeQL to handle bundle URLs without tags 2023-02-03 19:15:06 +00:00
Henry Mercer e0fc1c91b2 Add regression test for a bundle URL without a tag 2023-02-03 19:13:24 +00:00
Arthur Baars b95df0b2e7 Merge pull request #1516 from aibaars/auth-parameter
Supply authorization parameter to toolcache.downloadTool()
2023-02-03 20:02:59 +01:00
Arthur Baars 2fed02cbe2 Supply authorization parameter to toolcache.downloadTool()
Previously we supplied the authorization information via the 'headers'
parameter. This works fine, except in some cases when the request is
retried.
2023-02-03 14:56:00 +01:00
Angela P Wen 0b2a40fa4a Merge pull request #1515 from github/mergeback/v2.2.1-to-main-3ebbd71c
Mergeback v2.2.1 refs/heads/releases/v2 into main
2023-01-27 02:02:26 -08:00
github-actions[bot] 395ec04a8b Update checked-in dependencies 2023-01-27 09:37:04 +00:00
github-actions[bot] e1070bd101 Update changelog and version after v2.2.1 2023-01-27 09:30:07 +00:00
Angela P Wen 3ebbd71c74 Merge pull request #1514 from github/update-v2.2.1-4664f3969
Merge main into releases/v2
2023-01-27 01:28:20 -08:00
github-actions[bot] 2ae6e13cc3 Update changelog for v2.2.1 2023-01-27 09:06:39 +00:00
Angela P Wen 4664f39699 Ensure that tools_download_duration_ms is int (#1513) 2023-01-27 09:03:57 +00:00
Henry Mercer b2e16761f3 Merge pull request #1512 from github/mergeback/v2.2.0-to-main-436dbd91
Mergeback v2.2.0 refs/heads/releases/v2 into main
2023-01-26 17:44:11 +00:00
github-actions[bot] 592a896a53 Update checked-in dependencies 2023-01-26 16:40:56 +00:00
github-actions[bot] 4a6b5a54c2 Update changelog and version after v2.2.0 2023-01-26 16:39:29 +00:00
29 changed files with 670 additions and 352 deletions
+6
View File
@@ -33,6 +33,12 @@
"alphabetize": {"order": "asc"}, "alphabetize": {"order": "asc"},
"newlines-between": "always" "newlines-between": "always"
}], }],
"max-len": ["error", {
"code": 120,
"ignoreUrls": true,
"ignoreStrings": true,
"ignoreTemplateLiterals": true
}],
"no-async-foreach/no-async-foreach": "error", "no-async-foreach/no-async-foreach": "error",
"no-console": "off", "no-console": "off",
"no-sequences": "error", "no-sequences": "error",
+12
View File
@@ -1,5 +1,17 @@
# CodeQL Action Changelog # CodeQL Action Changelog
## [UNRELEASED]
No user facing changes.
## 2.2.2 - 06 Feb 2023
- Fix an issue where customers using the CodeQL Action with the [CodeQL Action sync tool](https://docs.github.com/en/enterprise-server@3.7/admin/code-security/managing-github-advanced-security-for-your-enterprise/configuring-code-scanning-for-your-appliance#configuring-codeql-analysis-on-a-server-without-internet-access) would not be able to obtain the CodeQL tools. [#1517](https://github.com/github/codeql-action/pull/1517)
## 2.2.1 - 27 Jan 2023
No user facing changes.
## 2.2.0 - 26 Jan 2023 ## 2.2.0 - 26 Jan 2023
- Improve stability when choosing the default version of CodeQL to use in code scanning workflow runs on Actions on GitHub.com. [#1475](https://github.com/github/codeql-action/pull/1475) - Improve stability when choosing the default version of CodeQL to use in code scanning workflow runs on Actions on GitHub.com. [#1475](https://github.com/github/codeql-action/pull/1475)
+1 -1
View File
@@ -1,6 +1,6 @@
# CodeQL Action # CodeQL Action
This action runs GitHub's industry-leading semantic code analysis engine, CodeQL, against a repository's source code to find security vulnerabilities. It then automatically uploads the results to GitHub so they can be displayed in the repository's security tab. CodeQL runs an extensible set of [queries](https://github.com/github/codeql), which have been developed by the community and the [GitHub Security Lab](https://securitylab.github.com/) to find common vulnerabilities in your code. This action runs GitHub's industry-leading semantic code analysis engine, [CodeQL](https://codeql.github.com/), against a repository's source code to find security vulnerabilities. It then automatically uploads the results to GitHub so they can be displayed in the repository's security tab. CodeQL runs an extensible set of [queries](https://github.com/github/codeql), which have been developed by the community and the [GitHub Security Lab](https://securitylab.github.com/) to find common vulnerabilities in your code.
For a list of recent changes, see the CodeQL Action's [changelog](CHANGELOG.md). For a list of recent changes, see the CodeQL Action's [changelog](CHANGELOG.md).
Generated
+5 -1
View File
@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result; return result;
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0; exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
const fs = __importStar(require("fs")); const fs = __importStar(require("fs"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner")); const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -94,6 +94,10 @@ exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
* --extractor-options-verbosity that we need. * --extractor-options-verbosity that we need.
*/ */
exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3"; exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
/**
* Versions 2.11.1+ of the CodeQL Bundle include a `security-experimental` built-in query suite for each language.
*/
exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/** /**
* Set up CodeQL CLI access. * Set up CodeQL CLI access.
* *
+1 -1
View File
File diff suppressed because one or more lines are too long
+67 -35
View File
@@ -97,7 +97,7 @@ ava_1.default.beforeEach(() => {
* @returns the download URL for the bundle. This can be passed to the tools parameter of * @returns the download URL for the bundle. This can be passed to the tools parameter of
* `codeql.setupCodeQL`. * `codeql.setupCodeQL`.
*/ */
function mockDownloadApi({ apiDetails = sampleApiDetails, isPinned, tagName, }) { function mockDownloadApi({ apiDetails = sampleApiDetails, isPinned, repo = "github/codeql-action", platformSpecific = true, tagName, }) {
const platform = process.platform === "win32" const platform = process.platform === "win32"
? "win64" ? "win64"
: process.platform === "linux" : process.platform === "linux"
@@ -105,7 +105,7 @@ function mockDownloadApi({ apiDetails = sampleApiDetails, isPinned, tagName, })
: "osx64"; : "osx64";
const baseUrl = apiDetails?.url ?? "https://example.com"; const baseUrl = apiDetails?.url ?? "https://example.com";
const relativeUrl = apiDetails const relativeUrl = apiDetails
? `/github/codeql-action/releases/download/${tagName}/codeql-bundle-${platform}.tar.gz` ? `/${repo}/releases/download/${tagName}/codeql-bundle${platformSpecific ? `-${platform}` : ""}.tar.gz`
: `/download/${tagName}/codeql-bundle.tar.gz`; : `/download/${tagName}/codeql-bundle.tar.gz`;
(0, nock_1.default)(baseUrl) (0, nock_1.default)(baseUrl)
.get(relativeUrl) .get(relativeUrl)
@@ -157,7 +157,7 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`)); t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`); t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
} }
t.is(toolcache.findAllVersions("CodeQL").length, 2); t.is(toolcache.findAllVersions("CodeQL").length, 2);
}); });
@@ -177,7 +177,7 @@ function mockApiDetails(apiDetails) {
t.assert(toolcache.find("CodeQL", "0.0.0-20200610")); t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610"); t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.not(result.toolsDownloadDurationMs, undefined); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
}); });
}); });
const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [ const EXPLICITLY_REQUESTED_BUNDLE_TEST_CASES = [
@@ -212,7 +212,7 @@ for (const { cliVersion, expectedToolcacheVersion, } of EXPLICITLY_REQUESTED_BUN
t.assert(toolcache.find("CodeQL", expectedToolcacheVersion)); t.assert(toolcache.find("CodeQL", expectedToolcacheVersion));
t.deepEqual(result.toolsVersion, cliVersion); t.deepEqual(result.toolsVersion, cliVersion);
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.not(result.toolsDownloadDurationMs, undefined); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
}); });
}); });
} }
@@ -302,7 +302,7 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
}, (0, logging_1.getRunnerLogger)(true), false); }, (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion); t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2); t.is(cachedVersions.length, 2);
}); });
@@ -322,46 +322,78 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
const result = await codeql.setupCodeQL("latest", sampleApiDetails, tmpDir, util.GitHubVariant.DOTCOM, false, SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false); const result = await codeql.setupCodeQL("latest", sampleApiDetails, tmpDir, util.GitHubVariant.DOTCOM, false, SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
t.deepEqual(result.toolsVersion, defaults.cliVersion); t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2); t.is(cachedVersions.length, 2);
}); });
}); });
(0, ava_1.default)("download codeql bundle from github ae endpoint", async (t) => { for (const isBundleVersionInUrl of [true, false]) {
const inclusionString = isBundleVersionInUrl
? "includes"
: "does not include";
(0, ava_1.default)(`download codeql bundle from github ae endpoint (URL ${inclusionString} bundle version)`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const bundleAssetID = 10;
const platform = process.platform === "win32"
? "win64"
: process.platform === "linux"
? "linux64"
: "osx64";
const codeQLBundleName = `codeql-bundle-${platform}.tar.gz`;
const eventualDownloadUrl = isBundleVersionInUrl
? `https://example.githubenterprise.com/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`
: `https://example.githubenterprise.com/api/v3/repos/github/codeql-action/releases/assets/${bundleAssetID}`;
(0, nock_1.default)("https://example.githubenterprise.com")
.get(`/api/v3/enterprise/code-scanning/codeql-bundle/find/${defaults.bundleVersion}`)
.reply(200, {
assets: { [codeQLBundleName]: bundleAssetID },
});
(0, nock_1.default)("https://example.githubenterprise.com")
.get(`/api/v3/enterprise/code-scanning/codeql-bundle/download/${bundleAssetID}`)
.reply(200, {
url: eventualDownloadUrl,
});
(0, nock_1.default)("https://example.githubenterprise.com")
.get(eventualDownloadUrl.replace("https://example.githubenterprise.com", ""))
.replyWithFile(200, path_1.default.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`));
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";
const result = await codeql.setupCodeQL(undefined, sampleGHAEApiDetails, tmpDir, util.GitHubVariant.GHAE, false, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
}, (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsSource, init_1.ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
});
}
(0, ava_1.default)("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t) => {
await util.withTmpDir(async (tmpDir) => { await util.withTmpDir(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir); (0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const bundleAssetID = 10; mockApiDetails(sampleApiDetails);
const platform = process.platform === "win32" sinon.stub(actionsUtil, "isRunningLocalAction").returns(true);
? "win64" const releasesApiMock = mockReleaseApi({
: process.platform === "linux" assetNames: ["cli-version-2.12.2.txt"],
? "linux64" tagName: "codeql-bundle-20230203",
: "osx64";
const codeQLBundleName = `codeql-bundle-${platform}.tar.gz`;
(0, nock_1.default)("https://example.githubenterprise.com")
.get(`/api/v3/enterprise/code-scanning/codeql-bundle/find/${defaults.bundleVersion}`)
.reply(200, {
assets: { [codeQLBundleName]: bundleAssetID },
}); });
(0, nock_1.default)("https://example.githubenterprise.com") mockDownloadApi({
.get(`/api/v3/enterprise/code-scanning/codeql-bundle/download/${bundleAssetID}`) repo: "dsp-testing/codeql-cli-nightlies",
.reply(200, { platformSpecific: false,
url: `https://example.githubenterprise.com/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`, tagName: "codeql-bundle-20230203",
}); });
(0, nock_1.default)("https://example.githubenterprise.com") const result = await codeql.setupCodeQL("https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz", sampleApiDetails, tmpDir, util.GitHubVariant.DOTCOM, false, SAMPLE_DEFAULT_CLI_VERSION, (0, logging_1.getRunnerLogger)(true), false);
.get(`/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`) t.is(result.toolsVersion, "0.0.0-20230203");
.replyWithFile(200, path_1.default.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`));
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";
const result = await codeql.setupCodeQL(undefined, sampleGHAEApiDetails, tmpDir, util.GitHubVariant.GHAE, false, {
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
}, (0, logging_1.getRunnerLogger)(true), false);
t.is(result.toolsSource, init_1.ToolsSource.Download); t.is(result.toolsSource, init_1.ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.true(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1); t.is(cachedVersions.length, 1);
t.is(cachedVersions[0], "0.0.0-20230203");
t.false(releasesApiMock.isDone());
}); });
}); });
(0, ava_1.default)("getExtraOptions works for explicit paths", (t) => { (0, ava_1.default)("getExtraOptions works for explicit paths", (t) => {
File diff suppressed because one or more lines are too long
+16 -3
View File
@@ -131,7 +131,11 @@ async function addDefaultQueries(codeQL, languages, resultMap) {
await runResolveQueries(codeQL, resultMap, suites, undefined); await runResolveQueries(codeQL, resultMap, suites, undefined);
} }
// The set of acceptable values for built-in suites from the codeql bundle // The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ["security-extended", "security-and-quality"]; const builtinSuites = [
"security-experimental",
"security-extended",
"security-and-quality",
];
/** /**
* Determine the set of queries associated with suiteName's suites and add them to resultMap. * Determine the set of queries associated with suiteName's suites and add them to resultMap.
* Throws an error if suiteName is not a valid builtin suite. * Throws an error if suiteName is not a valid builtin suite.
@@ -143,6 +147,12 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
if (!found) { if (!found) {
throw new Error(getQueryUsesInvalid(configFile, suiteName)); throw new Error(getQueryUsesInvalid(configFile, suiteName));
} }
if (suiteName === "security-experimental" &&
!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE))) {
throw new Error(`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`);
}
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is // If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
// pack, then add the ML-powered query pack so that we run ML-powered queries. // pack, then add the ML-powered query pack so that we run ML-powered queries.
@@ -151,7 +161,9 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
(process.platform !== "win32" || (process.platform !== "win32" ||
(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS))) && (await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS))) &&
languages.includes("javascript") && languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-experimental" ||
found === "security-extended" ||
found === "security-and-quality") &&
!packs.javascript?.some(isMlPoweredJsQueriesPack) && !packs.javascript?.some(isMlPoweredJsQueriesPack) &&
(await featureEnablement.getValue(feature_flags_1.Feature.MlPoweredQueriesEnabled, codeQL))) { (await featureEnablement.getValue(feature_flags_1.Feature.MlPoweredQueriesEnabled, codeQL))) {
if (!packs.javascript) { if (!packs.javascript) {
@@ -893,7 +905,8 @@ exports.parsePacks = parsePacks;
* Without a '+', an input value will override the corresponding value in the config file. * Without a '+', an input value will override the corresponding value in the config file.
* *
* @param inputValue The input value to process. * @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file, false if it should be appended. * @returns true if the input value should replace the corresponding value in the config file,
* false if it should be appended.
*/ */
function shouldCombine(inputValue) { function shouldCombine(inputValue) {
return !!inputValue?.trim().startsWith("+"); return !!inputValue?.trim().startsWith("+");
File diff suppressed because one or more lines are too long
+4 -1
View File
@@ -1014,7 +1014,7 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
// Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI. // Test that the ~0.1.0 version of ML-powered queries is run on v2.8.3 of the CLI.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.8.3", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.1.0"); (0, ava_1.default)(mlPoweredQueriesMacro, "2.8.3", true, undefined, "security-extended", process.platform === "win32" ? undefined : "~0.1.0");
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the // Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended` or `security-and-quality` query suite. // `security-extended`, `security-and-quality`, or `security-experimental` query suite.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined); (0, ava_1.default)(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on // Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
// versions of the CodeQL CLI prior to 2.9.0. // versions of the CodeQL CLI prior to 2.9.0.
@@ -1042,6 +1042,9 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL // Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.11.3+. // CLI 2.11.3+.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.11.3", true, undefined, "security-and-quality", "~0.4.0"); (0, ava_1.default)(mlPoweredQueriesMacro, "2.11.3", true, undefined, "security-and-quality", "~0.4.0");
// Test that ML-powered queries are run on all platforms running `security-experimental` on CodeQL
// CLI 2.12.1+.
(0, ava_1.default)(mlPoweredQueriesMacro, "2.12.1", true, undefined, "security-experimental", "~0.4.0");
const calculateAugmentationMacro = ava_1.default.macro({ const calculateAugmentationMacro = ava_1.default.macro({
exec: async (t, _title, rawPacksInput, rawQueriesInput, languages, expectedAugmentationProperties) => { exec: async (t, _title, rawPacksInput, rawQueriesInput, languages, expectedAugmentationProperties) => {
const actualAugmentationProperties = configUtils.calculateAugmentation(rawPacksInput, rawQueriesInput, languages); const actualAugmentationProperties = configUtils.calculateAugmentation(rawPacksInput, rawQueriesInput, languages);
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+163 -102
View File
@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod }; return (mod && mod.__esModule) ? mod : { "default": mod };
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.getCodeQLSource = exports.convertToSemVer = exports.getBundleVersionFromUrl = exports.tryFindCliVersionDotcomOnly = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0; exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.tryGetFallbackToolcacheVersion = exports.getCodeQLSource = exports.convertToSemVer = exports.tryGetBundleVersionFromUrl = exports.tryFindCliVersionDotcomOnly = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0;
const fs = __importStar(require("fs")); const fs = __importStar(require("fs"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks"); const perf_hooks_1 = require("perf_hooks");
@@ -211,21 +211,30 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger)
} }
return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
} }
function getBundleVersionFromTagName(tagName) { function tryGetBundleVersionFromTagName(tagName, logger) {
const match = tagName.match(/^codeql-bundle-(.*)$/); const match = tagName.match(/^codeql-bundle-(.*)$/);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error(`Malformed bundle tag name: ${tagName}. Bundle version could not be inferred`); logger.debug(`Could not determine bundle version from tag ${tagName}.`);
return undefined;
} }
return match[1]; return match[1];
} }
function getBundleVersionFromUrl(url) { function tryGetTagNameFromUrl(url, logger) {
const match = url.match(/\/(codeql-bundle-.*)\//); const match = url.match(/\/(codeql-bundle-.*)\//);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Bundle version could not be inferred`); logger.debug(`Could not determine tag name for URL ${url}.`);
return undefined;
} }
return getBundleVersionFromTagName(match[1]); return match[1];
} }
exports.getBundleVersionFromUrl = getBundleVersionFromUrl; function tryGetBundleVersionFromUrl(url, logger) {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
exports.tryGetBundleVersionFromUrl = tryGetBundleVersionFromUrl;
function convertToSemVer(version, logger) { function convertToSemVer(version, logger) {
if (!semver.valid(version)) { if (!semver.valid(version)) {
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`); logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
@@ -238,18 +247,10 @@ function convertToSemVer(version, logger) {
return s; return s;
} }
exports.convertToSemVer = convertToSemVer; exports.convertToSemVer = convertToSemVer;
async function getOrFindBundleTagName(version, logger) {
if (version.variant === util.GitHubVariant.DOTCOM) {
return await findCodeQLBundleTagDotcomOnly(version.cliVersion, logger);
}
else {
return version.tagName;
}
}
/** /**
* Look for a version of the CodeQL tools in the cache which could override the requested CLI version. * Look for a version of the CodeQL tools in the cache which could override the requested CLI version.
*/ */
async function findOverridingToolsInCache(requestedCliVersion, logger) { async function findOverridingToolsInCache(humanReadableVersion, logger) {
const candidates = toolcache const candidates = toolcache
.findAllVersions("CodeQL") .findAllVersions("CodeQL")
.filter(util_1.isGoodVersion) .filter(util_1.isGoodVersion)
@@ -260,7 +261,7 @@ async function findOverridingToolsInCache(requestedCliVersion, logger) {
.filter(({ folder }) => fs.existsSync(path.join(folder, "pinned-version"))); .filter(({ folder }) => fs.existsSync(path.join(folder, "pinned-version")));
if (candidates.length === 1) { if (candidates.length === 1) {
const candidate = candidates[0]; const candidate = candidates[0];
logger.debug(`CodeQL tools version ${candidate.version} in toolcache overriding version ${requestedCliVersion}.`); logger.debug(`CodeQL tools version ${candidate.version} in toolcache overriding version ${humanReadableVersion}.`);
return { return {
codeqlFolder: candidate.folder, codeqlFolder: candidate.folder,
sourceType: "toolcache", sourceType: "toolcache",
@@ -284,7 +285,8 @@ async function getCodeQLSource(toolsInput, bypassToolcache, defaultCliVersion, a
toolsVersion: "local", toolsVersion: "local",
}; };
} }
const forceLatestReason = /** The reason why the tools shipped with the Action have been forced. */
const forceShippedToolsReason =
// We use the special value of 'latest' to prioritize the version in the // We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version. // defaults over any pinned cached version.
toolsInput === "latest" toolsInput === "latest"
@@ -296,112 +298,160 @@ async function getCodeQLSource(toolsInput, bypassToolcache, defaultCliVersion, a
toolsInput === undefined && bypassToolcache toolsInput === undefined && bypassToolcache
? "a specific version of the CodeQL tools was not requested and the bypass toolcache feature is enabled" ? "a specific version of the CodeQL tools was not requested and the bypass toolcache feature is enabled"
: undefined; : undefined;
const forceLatest = forceLatestReason !== undefined; /** Whether the tools shipped with the Action, i.e. those in `defaults.json`, have been forced. */
if (forceLatest) { const forceShippedTools = forceShippedToolsReason !== undefined;
logger.debug(`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`); if (forceShippedTools) {
logger.info("Overriding the version of the CodeQL tools by the version shipped with the Action since " +
`${forceShippedToolsReason}.`);
} }
/** CLI version number, for example 2.12.1. */
let cliVersion;
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
let tagName;
/** /**
* The requested version is: * URL of the CodeQL bundle.
* *
* 1. The one in `defaults.json`, if forceLatest is true. * This does not always include a tag name.
* 2. The version specified by the tools input URL, if one was provided.
* 3. The default CLI version, otherwise.
* We include a `variant` property to let us verify using the type system that
* `tagName` is only undefined when the variant is Dotcom. This lets us ensure
* that we can always compute `tagName`, either by using the existing tag name
* on enterprise instances, or calling `findCodeQLBundleTagDotcomOnly` on
* Dotcom.
*/ */
const requestedVersion = forceLatest let url;
? // case 1 if (forceShippedTools) {
{ cliVersion = defaults.cliVersion;
cliVersion: defaults.cliVersion, tagName = defaults.bundleVersion;
syntheticCliVersion: defaults.cliVersion, }
tagName: defaults.bundleVersion, else if (toolsInput !== undefined) {
variant, // If a tools URL was provided, then use that.
} tagName = tryGetTagNameFromUrl(toolsInput, logger);
: toolsInput !== undefined url = toolsInput;
? // case 2 }
{ else {
syntheticCliVersion: convertToSemVer(getBundleVersionFromUrl(toolsInput), logger), // Otherwise, use the default CLI version passed in.
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`, cliVersion = defaultCliVersion.cliVersion;
url: toolsInput, tagName = defaultCliVersion["tagName"];
variant, }
} const bundleVersion = tagName && tryGetBundleVersionFromTagName(tagName, logger);
: // case 3 const humanReadableVersion = cliVersion ??
{ (bundleVersion && convertToSemVer(bundleVersion, logger)) ??
...defaultCliVersion, tagName ??
syntheticCliVersion: defaultCliVersion.cliVersion, url ??
}; "unknown";
// If we find the specified version, we always use that. logger.debug("Attempting to obtain CodeQL tools. " +
let codeqlFolder = toolcache.find("CodeQL", requestedVersion.syntheticCliVersion); `CLI version: ${cliVersion ?? "unknown"}, ` +
let tagName = requestedVersion["tagName"]; `bundle tag name: ${tagName ?? "unknown"}, ` +
if (!codeqlFolder) { `URL: ${url ?? "unspecified"}.`);
logger.debug("Didn't find a version of the CodeQL tools in the toolcache with a version number " + let codeqlFolder;
`exactly matching ${requestedVersion.syntheticCliVersion}.`); if (cliVersion) {
if (requestedVersion.cliVersion) { // If we find the specified CLI version, we always use that.
codeqlFolder = toolcache.find("CodeQL", cliVersion);
// Fall back to matching `x.y.z-<tagName>`.
if (!codeqlFolder) {
logger.debug("Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${cliVersion}.`);
const allVersions = toolcache.findAllVersions("CodeQL"); const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`); logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is // If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it. // the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) => version.startsWith(`${requestedVersion.cliVersion}-`)); const candidateVersions = allVersions.filter((version) => version.startsWith(`${cliVersion}-`));
if (candidateVersions.length === 1) { if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that."); logger.debug(`Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` +
"toolcache, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]); codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} }
else if (candidateVersions.length === 0) {
logger.debug(`Didn't find any versions of the CodeQL tools starting with ${cliVersion} ` +
`in the toolcache. Trying next fallback method.`);
}
else { else {
logger.debug("Did not find exactly one version of the CodeQL tools starting with the requested version."); logger.warning(`Found ${candidateVersions.length} versions of the CodeQL tools starting with ` +
`${cliVersion} in the toolcache, but at most one was expected.`);
logger.debug("Trying next fallback method.");
} }
} }
} }
if (!codeqlFolder && requestedVersion.cliVersion) { // Fall back to matching `0.0.0-<bundleVersion>`.
// Fall back to accepting a `0.0.0-<bundleVersion>` version if we didn't find the if (!codeqlFolder && (cliVersion || tagName)) {
// `x.y.z` version. This is to support old versions of the toolcache. if (cliVersion || tagName) {
// const fallbackVersion = await tryGetFallbackToolcacheVersion(cliVersion, tagName, variant, logger);
// If we are on Dotcom, we will make an HTTP request to the Releases API here if (fallbackVersion) {
// to find the tag name for the requested version. codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
tagName = }
tagName || (await getOrFindBundleTagName(requestedVersion, logger)); else {
const fallbackVersion = convertToSemVer(getBundleVersionFromTagName(tagName), logger); logger.debug("Could not determine a fallback toolcache version number for CodeQL tools version " +
logger.debug(`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL tools version ` + `${humanReadableVersion}.`);
`${requestedVersion.cliVersion}.`); }
codeqlFolder = toolcache.find("CodeQL", fallbackVersion); }
else {
logger.debug("Both the CLI version and the bundle version are unknown, so we will not be able to find " +
"the requested version of the CodeQL tools in the toolcache.");
}
}
if (codeqlFolder) {
logger.info(`Found CodeQL tools version ${humanReadableVersion} in the toolcache.`);
}
else {
logger.info(`Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`);
} }
if (codeqlFolder) { if (codeqlFolder) {
return { return {
codeqlFolder, codeqlFolder,
sourceType: "toolcache", sourceType: "toolcache",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
logger.debug(`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`);
// If we don't find the requested version on Enterprise, we may allow a // If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been // different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it). // specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) { if (variant !== util.GitHubVariant.DOTCOM &&
const result = await findOverridingToolsInCache(requestedVersion.syntheticCliVersion, logger); !forceShippedTools &&
!toolsInput) {
const result = await findOverridingToolsInCache(humanReadableVersion, logger);
if (result !== undefined) { if (result !== undefined) {
return result; return result;
} }
} }
if (!url) {
if (!tagName && cliVersion && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
else if (!tagName) {
throw new Error(`Could not obtain the requested version (${humanReadableVersion}) of the CodeQL tools ` +
"since we could not compute the tag name.");
}
url = await getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger);
}
return { return {
cliVersion: requestedVersion.cliVersion || undefined, bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
codeqlURL: requestedVersion["url"] || cliVersion,
(await getCodeQLBundleDownloadURL(tagName || codeqlURL: url,
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)), apiDetails, variant, logger)),
sourceType: "download", sourceType: "download",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
exports.getCodeQLSource = getCodeQLSource; exports.getCodeQLSource = getCodeQLSource;
async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, tempDir, logger) { /**
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
async function tryGetFallbackToolcacheVersion(cliVersion, tagName, variant, logger) {
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
if (cliVersion && !tagName && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
if (!tagName) {
return undefined;
}
const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger);
if (!bundleVersion) {
return undefined;
}
const fallbackVersion = convertToSemVer(bundleVersion, logger);
logger.debug(`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL version ` +
`${cliVersion ?? tagName}.`);
return fallbackVersion;
}
exports.tryGetFallbackToolcacheVersion = tryGetFallbackToolcacheVersion;
async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, variant, tempDir, logger) {
const parsedCodeQLURL = new URL(codeqlURL); const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search); const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers = { const headers = {
@@ -411,12 +461,13 @@ async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, t
// from the same GitHub instance the Action is running on. // from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom. // This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL. // We also don't want to send an authorization header if there's already a token provided in the URL.
let authorization = undefined;
if (searchParams.has("token")) { if (searchParams.has("token")) {
logger.debug("CodeQL tools URL contains an authorization token."); logger.debug("CodeQL tools URL contains an authorization token.");
} }
else if (codeqlURL.startsWith(`${apiDetails.url}/`)) { else if (codeqlURL.startsWith(`${apiDetails.url}/`)) {
logger.debug("Providing an authorization token to download CodeQL tools."); logger.debug("Providing an authorization token to download CodeQL tools.");
headers.authorization = `token ${apiDetails.auth}`; authorization = `token ${apiDetails.auth}`;
} }
else { else {
logger.debug("Downloading CodeQL tools without an authorization token."); logger.debug("Downloading CodeQL tools without an authorization token.");
@@ -425,16 +476,26 @@ async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, t
const dest = path.join(tempDir, (0, uuid_1.v4)()); const dest = path.join(tempDir, (0, uuid_1.v4)());
const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers); const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
const toolsDownloadStart = perf_hooks_1.performance.now(); const toolsDownloadStart = perf_hooks_1.performance.now();
const codeqlPath = await toolcache.downloadTool(codeqlURL, dest, undefined, finalHeaders); const codeqlPath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
const toolsDownloadDurationMs = perf_hooks_1.performance.now() - toolsDownloadStart; const toolsDownloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`); logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath); const codeqlExtracted = await toolcache.extractTar(codeqlPath);
const bundleVersion = getBundleVersionFromUrl(codeqlURL); const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);
return {
toolsVersion: maybeCliVersion ?? "unknown",
codeqlFolder: codeqlExtracted,
toolsDownloadDurationMs,
};
}
// Try to compute the CLI version for this bundle // Try to compute the CLI version for this bundle
const cliVersion = maybeCliVersion || if (maybeCliVersion === undefined &&
(variant === util.GitHubVariant.DOTCOM && variant === util.GitHubVariant.DOTCOM &&
(await tryFindCliVersionDotcomOnly(`codeql-bundle-${bundleVersion}`, logger))) || codeqlURL.includes(`/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/`)) {
undefined; maybeCliVersion = await tryFindCliVersionDotcomOnly(`codeql-bundle-${bundleVersion}`, logger);
}
// Include both the CLI version and the bundle version in the toolcache version number. That way // Include both the CLI version and the bundle version in the toolcache version number. That way
// if the user requests the same URL again, we can get it from the cache without having to call // if the user requests the same URL again, we can get it from the cache without having to call
// any of the Releases API. // any of the Releases API.
@@ -444,11 +505,11 @@ async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, t
// CLI release. In principle, it should be enough to just check that the CLI version isn't a // CLI release. In principle, it should be enough to just check that the CLI version isn't a
// pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`, // pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`,
// and we don't want these nightlies to override stable CLI versions in the toolcache. // and we don't want these nightlies to override stable CLI versions in the toolcache.
const toolcacheVersion = cliVersion && cliVersion.match(/^[0-9]+\.[0-9]+\.[0-9]+$/) const toolcacheVersion = maybeCliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)
? `${cliVersion}-${bundleVersion}` ? `${maybeCliVersion}-${bundleVersion}`
: convertToSemVer(bundleVersion, logger); : convertToSemVer(bundleVersion, logger);
return { return {
toolsVersion: cliVersion || toolcacheVersion, toolsVersion: maybeCliVersion ?? toolcacheVersion,
codeqlFolder: await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion), codeqlFolder: await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion),
toolsDownloadDurationMs, toolsDownloadDurationMs,
}; };
@@ -493,7 +554,7 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, bypas
toolsSource = init_1.ToolsSource.Toolcache; toolsSource = init_1.ToolsSource.Toolcache;
break; break;
case "download": { case "download": {
const result = await downloadCodeQL(source.codeqlURL, source.cliVersion, apiDetails, variant, tempDir, logger); const result = await downloadCodeQL(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, variant, tempDir, logger);
toolsVersion = result.toolsVersion; toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder; codeqlFolder = result.codeqlFolder;
toolsDownloadDurationMs = result.toolsDownloadDurationMs; toolsDownloadDurationMs = result.toolsDownloadDurationMs;
File diff suppressed because one or more lines are too long
+2 -1
View File
@@ -293,7 +293,8 @@ async function waitForProcessing(repositoryNwo, sarifID, logger, options = {
if (Date.now() > if (Date.now() >
statusCheckingStarted + STATUS_CHECK_TIMEOUT_MILLISECONDS) { statusCheckingStarted + STATUS_CHECK_TIMEOUT_MILLISECONDS) {
// If the analysis hasn't finished processing in the allotted time, we continue anyway rather than failing. // If the analysis hasn't finished processing in the allotted time, we continue anyway rather than failing.
// It's possible the analysis will eventually finish processing, but it's not worth spending more Actions time waiting. // It's possible the analysis will eventually finish processing, but it's not worth spending more
// Actions time waiting.
logger.warning("Timed out waiting for analysis to finish processing. Continuing."); logger.warning("Timed out waiting for analysis to finish processing. Continuing.");
break; break;
} }
File diff suppressed because one or more lines are too long
Generated
+2 -1
View File
@@ -691,7 +691,8 @@ exports.isHostedRunner = isHostedRunner;
* @param languagesInput Languages input from the workflow * @param languagesInput Languages input from the workflow
* @param repository The owner/name of the repository * @param repository The owner/name of the repository
* @param logger A logger * @param logger A logger
* @returns A boolean indicating whether or not the toolcache should be bypassed and the latest codeql should be downloaded. * @returns A boolean indicating whether or not the toolcache should be bypassed
* and the latest codeql should be downloaded.
*/ */
async function shouldBypassToolcache(featuresEnablement, codeqlUrl, languagesInput, repository, logger) { async function shouldBypassToolcache(featuresEnablement, codeqlUrl, languagesInput, repository, logger) {
// An explicit codeql url is specified, that means the toolcache will not be used. // An explicit codeql url is specified, that means the toolcache will not be used.
+1 -1
View File
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.2.0", "version": "2.2.3",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.2.0", "version": "2.2.3",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "codeql", "name": "codeql",
"version": "2.2.0", "version": "2.2.3",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@actions/artifact": "^1.1.0", "@actions/artifact": "^1.1.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "codeql", "name": "codeql",
"version": "2.2.0", "version": "2.2.3",
"private": true, "private": true,
"description": "CodeQL action", "description": "CodeQL action",
"scripts": { "scripts": {
+111 -54
View File
@@ -87,10 +87,14 @@ test.beforeEach(() => {
function mockDownloadApi({ function mockDownloadApi({
apiDetails = sampleApiDetails, apiDetails = sampleApiDetails,
isPinned, isPinned,
repo = "github/codeql-action",
platformSpecific = true,
tagName, tagName,
}: { }: {
apiDetails?: GitHubApiDetails; apiDetails?: GitHubApiDetails;
isPinned?: boolean; isPinned?: boolean;
repo?: string;
platformSpecific?: boolean;
tagName: string; tagName: string;
}): string { }): string {
const platform = const platform =
@@ -102,7 +106,9 @@ function mockDownloadApi({
const baseUrl = apiDetails?.url ?? "https://example.com"; const baseUrl = apiDetails?.url ?? "https://example.com";
const relativeUrl = apiDetails const relativeUrl = apiDetails
? `/github/codeql-action/releases/download/${tagName}/codeql-bundle-${platform}.tar.gz` ? `/${repo}/releases/download/${tagName}/codeql-bundle${
platformSpecific ? `-${platform}` : ""
}.tar.gz`
: `/download/${tagName}/codeql-bundle.tar.gz`; : `/download/${tagName}/codeql-bundle.tar.gz`;
nock(baseUrl) nock(baseUrl)
@@ -208,7 +214,7 @@ test("downloads and caches explicitly requested bundles that aren't in the toolc
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`)); t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
t.is(result.toolsVersion, `0.0.0-${version}`); t.is(result.toolsVersion, `0.0.0-${version}`);
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
} }
t.is(toolcache.findAllVersions("CodeQL").length, 2); t.is(toolcache.findAllVersions("CodeQL").length, 2);
@@ -241,7 +247,7 @@ test("downloads an explicitly requested bundle even if a different version is ca
t.assert(toolcache.find("CodeQL", "0.0.0-20200610")); t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
t.deepEqual(result.toolsVersion, "0.0.0-20200610"); t.deepEqual(result.toolsVersion, "0.0.0-20200610");
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.not(result.toolsDownloadDurationMs, undefined); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
}); });
}); });
@@ -293,7 +299,7 @@ for (const {
t.assert(toolcache.find("CodeQL", expectedToolcacheVersion)); t.assert(toolcache.find("CodeQL", expectedToolcacheVersion));
t.deepEqual(result.toolsVersion, cliVersion); t.deepEqual(result.toolsVersion, cliVersion);
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.not(result.toolsDownloadDurationMs, undefined); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
}); });
}); });
} }
@@ -428,7 +434,7 @@ for (const variant of [util.GitHubVariant.GHAE, util.GitHubVariant.GHES]) {
); );
t.deepEqual(result.toolsVersion, defaults.cliVersion); t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2); t.is(cachedVersions.length, 2);
@@ -461,76 +467,127 @@ test('downloads bundle if "latest" tools specified but not cached', async (t) =>
); );
t.deepEqual(result.toolsVersion, defaults.cliVersion); t.deepEqual(result.toolsVersion, defaults.cliVersion);
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2); t.is(cachedVersions.length, 2);
}); });
}); });
test("download codeql bundle from github ae endpoint", async (t) => { for (const isBundleVersionInUrl of [true, false]) {
const inclusionString = isBundleVersionInUrl
? "includes"
: "does not include";
test(`download codeql bundle from github ae endpoint (URL ${inclusionString} bundle version)`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
const bundleAssetID = 10;
const platform =
process.platform === "win32"
? "win64"
: process.platform === "linux"
? "linux64"
: "osx64";
const codeQLBundleName = `codeql-bundle-${platform}.tar.gz`;
const eventualDownloadUrl = isBundleVersionInUrl
? `https://example.githubenterprise.com/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`
: `https://example.githubenterprise.com/api/v3/repos/github/codeql-action/releases/assets/${bundleAssetID}`;
nock("https://example.githubenterprise.com")
.get(
`/api/v3/enterprise/code-scanning/codeql-bundle/find/${defaults.bundleVersion}`
)
.reply(200, {
assets: { [codeQLBundleName]: bundleAssetID },
});
nock("https://example.githubenterprise.com")
.get(
`/api/v3/enterprise/code-scanning/codeql-bundle/download/${bundleAssetID}`
)
.reply(200, {
url: eventualDownloadUrl,
});
nock("https://example.githubenterprise.com")
.get(
eventualDownloadUrl.replace(
"https://example.githubenterprise.com",
""
)
)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";
const result = await codeql.setupCodeQL(
undefined,
sampleGHAEApiDetails,
tmpDir,
util.GitHubVariant.GHAE,
false,
{
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
},
getRunnerLogger(true),
false
);
t.is(result.toolsSource, ToolsSource.Download);
t.assert(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1);
});
});
}
test("bundle URL from another repo is cached as 0.0.0-bundleVersion", async (t) => {
await util.withTmpDir(async (tmpDir) => { await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir); setupActionsVars(tmpDir, tmpDir);
const bundleAssetID = 10; mockApiDetails(sampleApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(true);
const platform = const releasesApiMock = mockReleaseApi({
process.platform === "win32" assetNames: ["cli-version-2.12.2.txt"],
? "win64" tagName: "codeql-bundle-20230203",
: process.platform === "linux" });
? "linux64" mockDownloadApi({
: "osx64"; repo: "dsp-testing/codeql-cli-nightlies",
const codeQLBundleName = `codeql-bundle-${platform}.tar.gz`; platformSpecific: false,
tagName: "codeql-bundle-20230203",
nock("https://example.githubenterprise.com") });
.get(
`/api/v3/enterprise/code-scanning/codeql-bundle/find/${defaults.bundleVersion}`
)
.reply(200, {
assets: { [codeQLBundleName]: bundleAssetID },
});
nock("https://example.githubenterprise.com")
.get(
`/api/v3/enterprise/code-scanning/codeql-bundle/download/${bundleAssetID}`
)
.reply(200, {
url: `https://example.githubenterprise.com/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`,
});
nock("https://example.githubenterprise.com")
.get(
`/github/codeql-action/releases/download/${defaults.bundleVersion}/${codeQLBundleName}`
)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
mockApiDetails(sampleGHAEApiDetails);
sinon.stub(actionsUtil, "isRunningLocalAction").returns(false);
process.env["GITHUB_ACTION_REPOSITORY"] = "github/codeql-action";
const result = await codeql.setupCodeQL( const result = await codeql.setupCodeQL(
undefined, "https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-20230203/codeql-bundle.tar.gz",
sampleGHAEApiDetails, sampleApiDetails,
tmpDir, tmpDir,
util.GitHubVariant.GHAE, util.GitHubVariant.DOTCOM,
false, false,
{ SAMPLE_DEFAULT_CLI_VERSION,
cliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant: util.GitHubVariant.GHAE,
},
getRunnerLogger(true), getRunnerLogger(true),
false false
); );
t.is(result.toolsVersion, "0.0.0-20230203");
t.is(result.toolsSource, ToolsSource.Download); t.is(result.toolsSource, ToolsSource.Download);
t.is(typeof result.toolsDownloadDurationMs, "number"); t.true(Number.isInteger(result.toolsDownloadDurationMs));
const cachedVersions = toolcache.findAllVersions("CodeQL"); const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 1); t.is(cachedVersions.length, 1);
t.is(cachedVersions[0], "0.0.0-20230203");
t.false(releasesApiMock.isDone());
}); });
}); });
+5
View File
@@ -278,6 +278,11 @@ export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
*/ */
export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3"; export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
/**
* Versions 2.11.1+ of the CodeQL Bundle include a `security-experimental` built-in query suite for each language.
*/
export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
/** /**
* Set up CodeQL CLI access. * Set up CodeQL CLI access.
* *
+11 -2
View File
@@ -1993,7 +1993,7 @@ test(
process.platform === "win32" ? undefined : "~0.1.0" process.platform === "win32" ? undefined : "~0.1.0"
); );
// Test that ML-powered queries aren't run when the user hasn't specified that we should run the // Test that ML-powered queries aren't run when the user hasn't specified that we should run the
// `security-extended` or `security-and-quality` query suite. // `security-extended`, `security-and-quality`, or `security-experimental` query suite.
test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined); test(mlPoweredQueriesMacro, "2.7.5", true, undefined, undefined, undefined);
// Test that ML-powered queries are run on non-Windows platforms running `security-extended` on // Test that ML-powered queries are run on non-Windows platforms running `security-extended` on
// versions of the CodeQL CLI prior to 2.9.0. // versions of the CodeQL CLI prior to 2.9.0.
@@ -2074,7 +2074,6 @@ test(
"security-extended", "security-extended",
"~0.4.0" "~0.4.0"
); );
// Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL // Test that ML-powered queries are run on all platforms running `security-and-quality` on CodeQL
// CLI 2.11.3+. // CLI 2.11.3+.
test( test(
@@ -2085,6 +2084,16 @@ test(
"security-and-quality", "security-and-quality",
"~0.4.0" "~0.4.0"
); );
// Test that ML-powered queries are run on all platforms running `security-experimental` on CodeQL
// CLI 2.12.1+.
test(
mlPoweredQueriesMacro,
"2.12.1",
true,
undefined,
"security-experimental",
"~0.4.0"
);
const calculateAugmentationMacro = test.macro({ const calculateAugmentationMacro = test.macro({
exec: async ( exec: async (
+24 -3
View File
@@ -10,6 +10,7 @@ import {
CodeQL, CodeQL,
CODEQL_VERSION_GHES_PACK_DOWNLOAD, CODEQL_VERSION_GHES_PACK_DOWNLOAD,
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS, CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS,
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE,
ResolveQueriesOutput, ResolveQueriesOutput,
} from "./codeql"; } from "./codeql";
import * as externalQueries from "./external-queries"; import * as externalQueries from "./external-queries";
@@ -380,7 +381,11 @@ async function addDefaultQueries(
} }
// The set of acceptable values for built-in suites from the codeql bundle // The set of acceptable values for built-in suites from the codeql bundle
const builtinSuites = ["security-extended", "security-and-quality"] as const; const builtinSuites = [
"security-experimental",
"security-extended",
"security-and-quality",
] as const;
/** /**
* Determine the set of queries associated with suiteName's suites and add them to resultMap. * Determine the set of queries associated with suiteName's suites and add them to resultMap.
@@ -401,6 +406,19 @@ async function addBuiltinSuiteQueries(
if (!found) { if (!found) {
throw new Error(getQueryUsesInvalid(configFile, suiteName)); throw new Error(getQueryUsesInvalid(configFile, suiteName));
} }
if (
suiteName === "security-experimental" &&
!(await codeQlVersionAbove(
codeQL,
CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE
))
) {
throw new Error(
`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`
);
}
// If we're running the JavaScript security-extended analysis (or a superset of it), the repo is // If we're running the JavaScript security-extended analysis (or a superset of it), the repo is
// opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query // opted into the ML-powered queries beta, and a user hasn't already added the ML-powered query
@@ -413,7 +431,9 @@ async function addBuiltinSuiteQueries(
CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS
))) && ))) &&
languages.includes("javascript") && languages.includes("javascript") &&
(found === "security-extended" || found === "security-and-quality") && (found === "security-experimental" ||
found === "security-extended" ||
found === "security-and-quality") &&
!packs.javascript?.some(isMlPoweredJsQueriesPack) && !packs.javascript?.some(isMlPoweredJsQueriesPack) &&
(await featureEnablement.getValue(Feature.MlPoweredQueriesEnabled, codeQL)) (await featureEnablement.getValue(Feature.MlPoweredQueriesEnabled, codeQL))
) { ) {
@@ -1630,7 +1650,8 @@ export function parsePacks(
* Without a '+', an input value will override the corresponding value in the config file. * Without a '+', an input value will override the corresponding value in the config file.
* *
* @param inputValue The input value to process. * @param inputValue The input value to process.
* @returns true if the input value should replace the corresponding value in the config file, false if it should be appended. * @returns true if the input value should replace the corresponding value in the config file,
* false if it should be appended.
*/ */
function shouldCombine(inputValue?: string): boolean { function shouldCombine(inputValue?: string): boolean {
return !!inputValue?.trim().startsWith("+"); return !!inputValue?.trim().startsWith("+");
+2 -1
View File
@@ -87,7 +87,8 @@ interface InitWithConfigStatusReport extends InitStatusReport {
interface InitToolsDownloadFields { interface InitToolsDownloadFields {
/** Time taken to download the bundle, in milliseconds. */ /** Time taken to download the bundle, in milliseconds. */
tools_download_duration_ms?: number; tools_download_duration_ms?: number;
/** Whether the relevant tools dotcom feature flags have been misconfigured. Only populated if we attempt to determine the default version based on the dotcom feature flags. */ /** Whether the relevant tools dotcom feature flags have been misconfigured.
* Only populated if we attempt to determine the default version based on the dotcom feature flags. */
tools_feature_flags_valid?: boolean; tools_feature_flags_valid?: boolean;
} }
+223 -133
View File
@@ -241,24 +241,36 @@ async function getCodeQLBundleDownloadURL(
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
} }
function getBundleVersionFromTagName(tagName: string): string { function tryGetBundleVersionFromTagName(
tagName: string,
logger: Logger
): string | undefined {
const match = tagName.match(/^codeql-bundle-(.*)$/); const match = tagName.match(/^codeql-bundle-(.*)$/);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error( logger.debug(`Could not determine bundle version from tag ${tagName}.`);
`Malformed bundle tag name: ${tagName}. Bundle version could not be inferred` return undefined;
);
} }
return match[1]; return match[1];
} }
export function getBundleVersionFromUrl(url: string): string { function tryGetTagNameFromUrl(url: string, logger: Logger): string | undefined {
const match = url.match(/\/(codeql-bundle-.*)\//); const match = url.match(/\/(codeql-bundle-.*)\//);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error( logger.debug(`Could not determine tag name for URL ${url}.`);
`Malformed tools url: ${url}. Bundle version could not be inferred` return undefined;
);
} }
return getBundleVersionFromTagName(match[1]); return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
} }
export function convertToSemVer(version: string, logger: Logger): string { export function convertToSemVer(version: string, logger: Logger): string {
@@ -291,6 +303,8 @@ type CodeQLToolsSource =
toolsVersion: string; toolsVersion: string;
} }
| { | {
/** Bundle version of the tools, if known. */
bundleVersion?: string;
/** CLI version of the tools, if known. */ /** CLI version of the tools, if known. */
cliVersion?: string; cliVersion?: string;
codeqlURL: string; codeqlURL: string;
@@ -299,22 +313,11 @@ type CodeQLToolsSource =
toolsVersion: string; toolsVersion: string;
}; };
async function getOrFindBundleTagName(
version: CodeQLDefaultVersionInfo,
logger: Logger
): Promise<string> {
if (version.variant === util.GitHubVariant.DOTCOM) {
return await findCodeQLBundleTagDotcomOnly(version.cliVersion, logger);
} else {
return version.tagName;
}
}
/** /**
* Look for a version of the CodeQL tools in the cache which could override the requested CLI version. * Look for a version of the CodeQL tools in the cache which could override the requested CLI version.
*/ */
async function findOverridingToolsInCache( async function findOverridingToolsInCache(
requestedCliVersion: string, humanReadableVersion: string,
logger: Logger logger: Logger
): Promise<CodeQLToolsSource | undefined> { ): Promise<CodeQLToolsSource | undefined> {
const candidates = toolcache const candidates = toolcache
@@ -329,7 +332,7 @@ async function findOverridingToolsInCache(
if (candidates.length === 1) { if (candidates.length === 1) {
const candidate = candidates[0]; const candidate = candidates[0];
logger.debug( logger.debug(
`CodeQL tools version ${candidate.version} in toolcache overriding version ${requestedCliVersion}.` `CodeQL tools version ${candidate.version} in toolcache overriding version ${humanReadableVersion}.`
); );
return { return {
codeqlFolder: candidate.folder, codeqlFolder: candidate.folder,
@@ -365,7 +368,8 @@ export async function getCodeQLSource(
}; };
} }
const forceLatestReason = /** The reason why the tools shipped with the Action have been forced. */
const forceShippedToolsReason =
// We use the special value of 'latest' to prioritize the version in the // We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version. // defaults over any pinned cached version.
toolsInput === "latest" toolsInput === "latest"
@@ -377,64 +381,67 @@ export async function getCodeQLSource(
toolsInput === undefined && bypassToolcache toolsInput === undefined && bypassToolcache
? "a specific version of the CodeQL tools was not requested and the bypass toolcache feature is enabled" ? "a specific version of the CodeQL tools was not requested and the bypass toolcache feature is enabled"
: undefined; : undefined;
const forceLatest = forceLatestReason !== undefined; /** Whether the tools shipped with the Action, i.e. those in `defaults.json`, have been forced. */
if (forceLatest) { const forceShippedTools = forceShippedToolsReason !== undefined;
logger.debug( if (forceShippedTools) {
`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.` logger.info(
"Overriding the version of the CodeQL tools by the version shipped with the Action since " +
`${forceShippedToolsReason}.`
); );
} }
/** CLI version number, for example 2.12.1. */
let cliVersion: string | undefined;
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
let tagName: string | undefined;
/** /**
* The requested version is: * URL of the CodeQL bundle.
* *
* 1. The one in `defaults.json`, if forceLatest is true. * This does not always include a tag name.
* 2. The version specified by the tools input URL, if one was provided.
* 3. The default CLI version, otherwise.
* We include a `variant` property to let us verify using the type system that
* `tagName` is only undefined when the variant is Dotcom. This lets us ensure
* that we can always compute `tagName`, either by using the existing tag name
* on enterprise instances, or calling `findCodeQLBundleTagDotcomOnly` on
* Dotcom.
*/ */
const requestedVersion = forceLatest let url: string | undefined;
? // case 1
{
cliVersion: defaults.cliVersion,
syntheticCliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
}
: toolsInput !== undefined
? // case 2
{
syntheticCliVersion: convertToSemVer(
getBundleVersionFromUrl(toolsInput),
logger
),
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`,
url: toolsInput,
variant,
}
: // case 3
{
...defaultCliVersion,
syntheticCliVersion: defaultCliVersion.cliVersion,
};
// If we find the specified version, we always use that. if (forceShippedTools) {
let codeqlFolder = toolcache.find( cliVersion = defaults.cliVersion;
"CodeQL", tagName = defaults.bundleVersion;
requestedVersion.syntheticCliVersion } else if (toolsInput !== undefined) {
// If a tools URL was provided, then use that.
tagName = tryGetTagNameFromUrl(toolsInput, logger);
url = toolsInput;
} else {
// Otherwise, use the default CLI version passed in.
cliVersion = defaultCliVersion.cliVersion;
tagName = defaultCliVersion["tagName"];
}
const bundleVersion =
tagName && tryGetBundleVersionFromTagName(tagName, logger);
const humanReadableVersion =
cliVersion ??
(bundleVersion && convertToSemVer(bundleVersion, logger)) ??
tagName ??
url ??
"unknown";
logger.debug(
"Attempting to obtain CodeQL tools. " +
`CLI version: ${cliVersion ?? "unknown"}, ` +
`bundle tag name: ${tagName ?? "unknown"}, ` +
`URL: ${url ?? "unspecified"}.`
); );
let tagName: string | undefined = requestedVersion["tagName"];
if (!codeqlFolder) { let codeqlFolder;
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " + if (cliVersion) {
`exactly matching ${requestedVersion.syntheticCliVersion}.` // If we find the specified CLI version, we always use that.
); codeqlFolder = toolcache.find("CodeQL", cliVersion);
if (requestedVersion.cliVersion) {
// Fall back to matching `x.y.z-<tagName>`.
if (!codeqlFolder) {
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${cliVersion}.`
);
const allVersions = toolcache.findAllVersions("CodeQL"); const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug( logger.debug(
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
@@ -444,55 +451,82 @@ export async function getCodeQLSource(
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is // If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it. // the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) => const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${requestedVersion.cliVersion}-`) version.startsWith(`${cliVersion}-`)
); );
if (candidateVersions.length === 1) { if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else {
logger.debug( logger.debug(
"Did not find exactly one version of the CodeQL tools starting with the requested version." `Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` +
"toolcache, using that."
); );
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else if (candidateVersions.length === 0) {
logger.debug(
`Didn't find any versions of the CodeQL tools starting with ${cliVersion} ` +
`in the toolcache. Trying next fallback method.`
);
} else {
logger.warning(
`Found ${candidateVersions.length} versions of the CodeQL tools starting with ` +
`${cliVersion} in the toolcache, but at most one was expected.`
);
logger.debug("Trying next fallback method.");
} }
} }
} }
if (!codeqlFolder && requestedVersion.cliVersion) { // Fall back to matching `0.0.0-<bundleVersion>`.
// Fall back to accepting a `0.0.0-<bundleVersion>` version if we didn't find the if (!codeqlFolder && (cliVersion || tagName)) {
// `x.y.z` version. This is to support old versions of the toolcache. if (cliVersion || tagName) {
// const fallbackVersion = await tryGetFallbackToolcacheVersion(
// If we are on Dotcom, we will make an HTTP request to the Releases API here cliVersion,
// to find the tag name for the requested version. tagName,
tagName = variant,
tagName || (await getOrFindBundleTagName(requestedVersion, logger)); logger
const fallbackVersion = convertToSemVer( );
getBundleVersionFromTagName(tagName), if (fallbackVersion) {
logger codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} else {
logger.debug(
"Could not determine a fallback toolcache version number for CodeQL tools version " +
`${humanReadableVersion}.`
);
}
} else {
logger.debug(
"Both the CLI version and the bundle version are unknown, so we will not be able to find " +
"the requested version of the CodeQL tools in the toolcache."
);
}
}
if (codeqlFolder) {
logger.info(
`Found CodeQL tools version ${humanReadableVersion} in the toolcache.`
); );
logger.debug( } else {
`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL tools version ` + logger.info(
`${requestedVersion.cliVersion}.` `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`
); );
codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} }
if (codeqlFolder) { if (codeqlFolder) {
return { return {
codeqlFolder, codeqlFolder,
sourceType: "toolcache", sourceType: "toolcache",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
logger.debug(
`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`
);
// If we don't find the requested version on Enterprise, we may allow a // If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been // different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it). // specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) { if (
variant !== util.GitHubVariant.DOTCOM &&
!forceShippedTools &&
!toolsInput
) {
const result = await findOverridingToolsInCache( const result = await findOverridingToolsInCache(
requestedVersion.syntheticCliVersion, humanReadableVersion,
logger logger
); );
if (result !== undefined) { if (result !== undefined) {
@@ -500,30 +534,66 @@ export async function getCodeQLSource(
} }
} }
if (!url) {
if (!tagName && cliVersion && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
} else if (!tagName) {
throw new Error(
`Could not obtain the requested version (${humanReadableVersion}) of the CodeQL tools ` +
"since we could not compute the tag name."
);
}
url = await getCodeQLBundleDownloadURL(
tagName,
apiDetails,
variant,
logger
);
}
return { return {
cliVersion: requestedVersion.cliVersion || undefined, bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
codeqlURL: cliVersion,
requestedVersion["url"] || codeqlURL: url,
(await getCodeQLBundleDownloadURL(
tagName ||
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)),
apiDetails,
variant,
logger
)),
sourceType: "download", sourceType: "download",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
/**
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
export async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string | undefined,
variant: util.GitHubVariant,
logger: Logger
): Promise<string | undefined> {
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
if (cliVersion && !tagName && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
if (!tagName) {
return undefined;
}
const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger);
if (!bundleVersion) {
return undefined;
}
const fallbackVersion = convertToSemVer(bundleVersion, logger);
logger.debug(
`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL version ` +
`${cliVersion ?? tagName}.`
);
return fallbackVersion;
}
export async function downloadCodeQL( export async function downloadCodeQL(
codeqlURL: string, codeqlURL: string,
maybeBundleVersion: string | undefined,
maybeCliVersion: string | undefined, maybeCliVersion: string | undefined,
apiDetails: api.GitHubApiDetails, apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant, variant: util.GitHubVariant,
@@ -543,11 +613,12 @@ export async function downloadCodeQL(
// from the same GitHub instance the Action is running on. // from the same GitHub instance the Action is running on.
// This avoids leaking Enterprise tokens to dotcom. // This avoids leaking Enterprise tokens to dotcom.
// We also don't want to send an authorization header if there's already a token provided in the URL. // We also don't want to send an authorization header if there's already a token provided in the URL.
let authorization: string | undefined = undefined;
if (searchParams.has("token")) { if (searchParams.has("token")) {
logger.debug("CodeQL tools URL contains an authorization token."); logger.debug("CodeQL tools URL contains an authorization token.");
} else if (codeqlURL.startsWith(`${apiDetails.url}/`)) { } else if (codeqlURL.startsWith(`${apiDetails.url}/`)) {
logger.debug("Providing an authorization token to download CodeQL tools."); logger.debug("Providing an authorization token to download CodeQL tools.");
headers.authorization = `token ${apiDetails.auth}`; authorization = `token ${apiDetails.auth}`;
} else { } else {
logger.debug("Downloading CodeQL tools without an authorization token."); logger.debug("Downloading CodeQL tools without an authorization token.");
} }
@@ -565,25 +636,44 @@ export async function downloadCodeQL(
const codeqlPath = await toolcache.downloadTool( const codeqlPath = await toolcache.downloadTool(
codeqlURL, codeqlURL,
dest, dest,
undefined, authorization,
finalHeaders finalHeaders
); );
const toolsDownloadDurationMs = performance.now() - toolsDownloadStart; const toolsDownloadDurationMs = Math.round(
performance.now() - toolsDownloadStart
);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`); logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath); const codeqlExtracted = await toolcache.extractTar(codeqlPath);
const bundleVersion = getBundleVersionFromUrl(codeqlURL); const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug(
"Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`
);
return {
toolsVersion: maybeCliVersion ?? "unknown",
codeqlFolder: codeqlExtracted,
toolsDownloadDurationMs,
};
}
// Try to compute the CLI version for this bundle // Try to compute the CLI version for this bundle
const cliVersion: string | undefined = if (
maybeCliVersion || maybeCliVersion === undefined &&
(variant === util.GitHubVariant.DOTCOM && variant === util.GitHubVariant.DOTCOM &&
(await tryFindCliVersionDotcomOnly( codeqlURL.includes(`/${CODEQL_DEFAULT_ACTION_REPOSITORY}/`)
`codeql-bundle-${bundleVersion}`, ) {
logger maybeCliVersion = await tryFindCliVersionDotcomOnly(
))) || `codeql-bundle-${bundleVersion}`,
undefined; logger
);
}
// Include both the CLI version and the bundle version in the toolcache version number. That way // Include both the CLI version and the bundle version in the toolcache version number. That way
// if the user requests the same URL again, we can get it from the cache without having to call // if the user requests the same URL again, we can get it from the cache without having to call
// any of the Releases API. // any of the Releases API.
@@ -593,12 +683,11 @@ export async function downloadCodeQL(
// CLI release. In principle, it should be enough to just check that the CLI version isn't a // CLI release. In principle, it should be enough to just check that the CLI version isn't a
// pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`, // pre-release, but the version numbers of CodeQL nightlies have the format `x.y.z+<timestamp>`,
// and we don't want these nightlies to override stable CLI versions in the toolcache. // and we don't want these nightlies to override stable CLI versions in the toolcache.
const toolcacheVersion = const toolcacheVersion = maybeCliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)
cliVersion && cliVersion.match(/^[0-9]+\.[0-9]+\.[0-9]+$/) ? `${maybeCliVersion}-${bundleVersion}`
? `${cliVersion}-${bundleVersion}` : convertToSemVer(bundleVersion, logger);
: convertToSemVer(bundleVersion, logger);
return { return {
toolsVersion: cliVersion || toolcacheVersion, toolsVersion: maybeCliVersion ?? toolcacheVersion,
codeqlFolder: await toolcache.cacheDir( codeqlFolder: await toolcache.cacheDir(
codeqlExtracted, codeqlExtracted,
"CodeQL", "CodeQL",
@@ -672,6 +761,7 @@ export async function setupCodeQLBundle(
case "download": { case "download": {
const result = await downloadCodeQL( const result = await downloadCodeQL(
source.codeqlURL, source.codeqlURL,
source.bundleVersion,
source.cliVersion, source.cliVersion,
apiDetails, apiDetails,
variant, variant,
+2 -1
View File
@@ -416,7 +416,8 @@ export async function waitForProcessing(
statusCheckingStarted + STATUS_CHECK_TIMEOUT_MILLISECONDS statusCheckingStarted + STATUS_CHECK_TIMEOUT_MILLISECONDS
) { ) {
// If the analysis hasn't finished processing in the allotted time, we continue anyway rather than failing. // If the analysis hasn't finished processing in the allotted time, we continue anyway rather than failing.
// It's possible the analysis will eventually finish processing, but it's not worth spending more Actions time waiting. // It's possible the analysis will eventually finish processing, but it's not worth spending more
// Actions time waiting.
logger.warning( logger.warning(
"Timed out waiting for analysis to finish processing. Continuing." "Timed out waiting for analysis to finish processing. Continuing."
); );
+2 -1
View File
@@ -808,7 +808,8 @@ export function isHostedRunner() {
* @param languagesInput Languages input from the workflow * @param languagesInput Languages input from the workflow
* @param repository The owner/name of the repository * @param repository The owner/name of the repository
* @param logger A logger * @param logger A logger
* @returns A boolean indicating whether or not the toolcache should be bypassed and the latest codeql should be downloaded. * @returns A boolean indicating whether or not the toolcache should be bypassed
* and the latest codeql should be downloaded.
*/ */
export async function shouldBypassToolcache( export async function shouldBypassToolcache(
featuresEnablement: FeatureEnablement, featuresEnablement: FeatureEnablement,