mirror of
https://github.com/github/codeql-action.git
synced 2026-08-06 13:13:45 -05:00
Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 794f5bc385 | |||
| 54109818e0 | |||
| 99caaa8b90 | |||
| 6117bb503a | |||
| af767ec1f6 | |||
| 7d9249f5a5 | |||
| 8ebf1091b0 | |||
| bdf39710a2 | |||
| 74cfae9be6 | |||
| 47a0a833bb | |||
| 6a90bf1f54 | |||
| c5995f544d | |||
| 76c44396d3 | |||
| fad141fa6c | |||
| 7d82f1132f | |||
| 37bdbde050 | |||
| 5595ccaf91 | |||
| ec9c75796a | |||
| 45c8742e17 | |||
| 6a9359a1bd | |||
| 065cdc0394 | |||
| f99dd5aeee | |||
| 1804b211a3 | |||
| 3020a2f462 | |||
| 93c3a5a40b | |||
| d1ba80a13d | |||
| e74600b0d9 | |||
| 266c7bdbd2 | |||
| daa7fe6fba | |||
| 1cd4d01d58 | |||
| d2bfc30bc3 | |||
| 68028fcb16 | |||
| c29563eeaa | |||
| 155e522997 | |||
| 2d3b351ea6 | |||
| 780c00da3f | |||
| 13d4882649 | |||
| e55a57b808 | |||
| 51d51e8121 | |||
| e893985e8b | |||
| eb692f8b49 | |||
| aac07d2a41 | |||
| de57c4a441 |
@@ -63,7 +63,7 @@ jobs:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install Java
|
||||
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
|
||||
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
|
||||
with:
|
||||
java-version: ${{ inputs.java-version || '17' }}
|
||||
distribution: temurin
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ jobs:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
- name: Install Java
|
||||
uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0
|
||||
uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0
|
||||
with:
|
||||
java-version: ${{ inputs.java-version || '17' }}
|
||||
distribution: temurin
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: github/enterprise-releases
|
||||
token: ${{ secrets.ENTERPRISE_RELEASE_TOKEN }}
|
||||
token: ${{ secrets.CODEQL_CI_ENTERPRISE_RELEASE_PAT }}
|
||||
path: ${{ github.workspace }}/enterprise-releases/
|
||||
sparse-checkout: releases.json
|
||||
|
||||
|
||||
@@ -6,6 +6,14 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
|
||||
|
||||
No user facing changes.
|
||||
|
||||
## 4.37.6 - 04 Aug 2026
|
||||
|
||||
- Changed the default filepath for the new remote file address format that was introduced in CodeQL Action 4.37.0 / 3.37.0 to `.github/codeql-config.yml` to align it with the suggested path that is used elsewhere. [#4070](https://github.com/github/codeql-action/pull/4070)
|
||||
|
||||
## 4.37.5 - 03 Aug 2026
|
||||
|
||||
- Fixed a bug where a network error while streaming the download of the CodeQL bundle could terminate the `init` Action instead of falling back to downloading the bundle before extracting it. [#4061](https://github.com/github/codeql-action/pull/4061)
|
||||
|
||||
## 4.37.4 - 29 Jul 2026
|
||||
|
||||
- This version of the CodeQL Action adds support for the `tools` input for the `codeql-action/init` step to be specified using a `github-codeql-tools` [repository property](https://docs.github.com/en/organizations/managing-organization-settings/managing-custom-properties-for-repositories-in-your-organization). This feature will gradually be rolled out following the release of this version. Once rolled out, this allows for the CodeQL CLI version that is used in GitHub-managed workflows, such as Default Setup, to be set to a custom value. For example, customers who run into issues with rate limits when a new CodeQL CLI version is released can set the value to `toolcache` to always use the CodeQL CLI version that is available in the runner toolcache. For Advanced Setup workflows, the value provided for `tools` in the workflow definition always takes precedence unless the value of the repository property starts with `!`. [#4037](https://github.com/github/codeql-action/pull/4037)
|
||||
|
||||
Generated
+850
-547
File diff suppressed because it is too large
Load Diff
Generated
+31
-31
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "codeql",
|
||||
"version": "4.37.5",
|
||||
"version": "4.37.7",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "codeql",
|
||||
"version": "4.37.5",
|
||||
"version": "4.37.7",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"pr-checks"
|
||||
@@ -31,7 +31,7 @@
|
||||
"follow-redirects": "^1.16.0",
|
||||
"get-folder-size": "^5.0.0",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^5.2.1",
|
||||
"js-yaml": "^5.2.2",
|
||||
"jsonschema": "1.5.0",
|
||||
"long": "^5.3.2",
|
||||
"node-forge": "^1.4.0",
|
||||
@@ -61,7 +61,7 @@
|
||||
"eslint-plugin-jsdoc": "^62.9.0",
|
||||
"eslint-plugin-no-async-foreach": "^0.1.1",
|
||||
"glob": "^13.0.6",
|
||||
"globals": "^17.7.0",
|
||||
"globals": "^17.8.0",
|
||||
"nock": "^14.0.16",
|
||||
"sinon": "^22.1.0",
|
||||
"typescript": "^6.0.3",
|
||||
@@ -374,9 +374,9 @@
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@actions/artifact/node_modules/brace-expansion": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz",
|
||||
"integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==",
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
|
||||
"integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0"
|
||||
@@ -2843,9 +2843,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz",
|
||||
"integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==",
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -3864,9 +3864,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.16",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz",
|
||||
"integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==",
|
||||
"version": "1.1.18",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
|
||||
"integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^1.0.0",
|
||||
@@ -5115,16 +5115,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-plugin-import-x/node_modules/brace-expansion": {
|
||||
"version": "5.0.7",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
|
||||
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/eslint-plugin-import-x/node_modules/minimatch": {
|
||||
@@ -6111,15 +6111,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/glob/node_modules/brace-expansion": {
|
||||
"version": "5.0.7",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
|
||||
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/glob/node_modules/minimatch": {
|
||||
@@ -6138,9 +6138,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/globals": {
|
||||
"version": "17.7.0",
|
||||
"resolved": "https://registry.npmjs.org/globals/-/globals-17.7.0.tgz",
|
||||
"integrity": "sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==",
|
||||
"version": "17.8.0",
|
||||
"resolved": "https://registry.npmjs.org/globals/-/globals-17.8.0.tgz",
|
||||
"integrity": "sha512-Zz/LMDZScFmkakeL2cTHzf+PbWKdpU3uclqkZT7TjDG58j5WPt0PpA+n9uPI24fZtlw07q0OtEi84K+umsRzqQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -6981,9 +6981,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/js-yaml": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.1.tgz",
|
||||
"integrity": "sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==",
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.2.2.tgz",
|
||||
"integrity": "sha512-dayzUzKkJ1MkuUtZglSebU43utNXH0OWQByK9rKOOuYIO8M5TV1y+n8ALMdG0rdzBnfNkOmZEqrURepb0ejqBw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
@@ -8090,15 +8090,15 @@
|
||||
}
|
||||
},
|
||||
"node_modules/readdir-glob/node_modules/brace-expansion": {
|
||||
"version": "5.0.7",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz",
|
||||
"integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==",
|
||||
"version": "5.0.9",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz",
|
||||
"integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
"node": "20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/readdir-glob/node_modules/minimatch": {
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "codeql",
|
||||
"version": "4.37.5",
|
||||
"version": "4.37.7",
|
||||
"private": true,
|
||||
"description": "CodeQL action",
|
||||
"scripts": {
|
||||
@@ -39,7 +39,7 @@
|
||||
"follow-redirects": "^1.16.0",
|
||||
"get-folder-size": "^5.0.0",
|
||||
"https-proxy-agent": "^7.0.6",
|
||||
"js-yaml": "^5.2.1",
|
||||
"js-yaml": "^5.2.2",
|
||||
"jsonschema": "1.5.0",
|
||||
"long": "^5.3.2",
|
||||
"node-forge": "^1.4.0",
|
||||
@@ -69,7 +69,7 @@
|
||||
"eslint-plugin-jsdoc": "^62.9.0",
|
||||
"eslint-plugin-no-async-foreach": "^0.1.1",
|
||||
"glob": "^13.0.6",
|
||||
"globals": "^17.7.0",
|
||||
"globals": "^17.8.0",
|
||||
"nock": "^14.0.16",
|
||||
"sinon": "^22.1.0",
|
||||
"typescript": "^6.0.3",
|
||||
|
||||
+2
-2
@@ -253,8 +253,8 @@ const languageSetups: LanguageSetups = {
|
||||
name: "Install Java",
|
||||
uses: pinnedUses(
|
||||
"actions/setup-java",
|
||||
"03ad4de0992f5dab5e18fcb136590ce7c4a0ac95",
|
||||
"v5.6.0",
|
||||
"b6effb05e454b25005698d916606bdc6ffcbf961",
|
||||
"v5.7.0",
|
||||
),
|
||||
with: {
|
||||
"java-version": `\${{ inputs.java-version || '${defaultLanguageVersions.java}' }}`,
|
||||
|
||||
+98
-24
@@ -1295,33 +1295,12 @@ checkOverlayEnablementMacro.serial(
|
||||
);
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
"No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled",
|
||||
"Overlay-base database on default branch if runner disk space is above the default limit",
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 5_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
|
||||
},
|
||||
);
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
"Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled",
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
@@ -1336,7 +1315,7 @@ checkOverlayEnablementMacro.serial(
|
||||
);
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
"No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled",
|
||||
"No overlay-base database on default branch if runner disk space is below the default limit",
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
@@ -1345,7 +1324,102 @@ checkOverlayEnablementMacro.serial(
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 15_000_000_000,
|
||||
numAvailableBytes: 10_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
|
||||
},
|
||||
);
|
||||
|
||||
// Check that each feature flag lowers the limit to the threshold that its name
|
||||
// declares. Both sides of the boundary are needed to pin the threshold down: a
|
||||
// mapping to a lower value would still pass the case at the limit, and one to a
|
||||
// higher value would still fail the case below it.
|
||||
for (const [feature, thresholdGb] of [
|
||||
[Feature.OverlayAnalysisMinDisk8Gb, 8],
|
||||
[Feature.OverlayAnalysisMinDisk9Gb, 9],
|
||||
[Feature.OverlayAnalysisMinDisk10Gb, 10],
|
||||
[Feature.OverlayAnalysisMinDisk11Gb, 11],
|
||||
[Feature.OverlayAnalysisMinDisk12Gb, 12],
|
||||
[Feature.OverlayAnalysisMinDisk13Gb, 13],
|
||||
] as Array<[Feature, number]>) {
|
||||
const features = [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
feature,
|
||||
];
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
`Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`,
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features,
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: thresholdGb * 1_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
useOverlayDatabaseCaching: true,
|
||||
},
|
||||
);
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
`No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`,
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features,
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
disabledReason: OverlayDisabledReason.InsufficientDiskSpace,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
"Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag",
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisMinDisk9Gb,
|
||||
Feature.OverlayAnalysisMinDisk12Gb,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 9_000_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
overlayDatabaseMode: OverlayDatabaseMode.OverlayBase,
|
||||
useOverlayDatabaseCaching: true,
|
||||
},
|
||||
);
|
||||
|
||||
checkOverlayEnablementMacro.serial(
|
||||
"No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag",
|
||||
{
|
||||
languages: [BuiltInLanguage.javascript],
|
||||
features: [
|
||||
Feature.OverlayAnalysis,
|
||||
Feature.OverlayAnalysisCodeScanningJavascript,
|
||||
Feature.OverlayAnalysisMinDisk9Gb,
|
||||
Feature.OverlayAnalysisMinDisk12Gb,
|
||||
],
|
||||
isDefaultBranch: true,
|
||||
diskUsage: {
|
||||
numAvailableBytes: 8_500_000_000,
|
||||
numTotalBytes: 100_000_000_000,
|
||||
},
|
||||
},
|
||||
|
||||
+49
-24
@@ -48,7 +48,7 @@ import {
|
||||
import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import * as errorMessages from "./error-messages";
|
||||
import { Feature, FeatureEnablement } from "./feature-flags";
|
||||
import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags";
|
||||
import {
|
||||
RepositoryProperties,
|
||||
RepositoryPropertyName,
|
||||
@@ -101,19 +101,28 @@ export { type Config } from "./config/action-config";
|
||||
* whether to perform overlay analysis, then the action will not perform overlay
|
||||
* analysis unless overlay analysis has been explicitly enabled via environment
|
||||
* variable.
|
||||
*
|
||||
* This threshold can be lowered by the feature flags in
|
||||
* `OVERLAY_MINIMUM_DISK_SPACE_FEATURES`.
|
||||
*/
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000;
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES =
|
||||
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000;
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000;
|
||||
|
||||
/**
|
||||
* The v2 minimum available disk space (in MB) required to perform overlay
|
||||
* analysis. This is a lower threshold than the v1 limit, allowing overlay
|
||||
* analysis to run on runners with less available disk space.
|
||||
* Feature flags that lower the minimum available disk space required to perform
|
||||
* overlay analysis, paired with the threshold (in MB) that each one enables.
|
||||
*
|
||||
* If several of these are enabled, the lowest threshold takes effect.
|
||||
*/
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000;
|
||||
const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES =
|
||||
OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000;
|
||||
const OVERLAY_MINIMUM_DISK_SPACE_FEATURES: ReadonlyArray<
|
||||
[FeatureWithoutCLI, number]
|
||||
> = [
|
||||
[Feature.OverlayAnalysisMinDisk8Gb, 8000],
|
||||
[Feature.OverlayAnalysisMinDisk9Gb, 9000],
|
||||
[Feature.OverlayAnalysisMinDisk10Gb, 10000],
|
||||
[Feature.OverlayAnalysisMinDisk11Gb, 11000],
|
||||
[Feature.OverlayAnalysisMinDisk12Gb, 12000],
|
||||
[Feature.OverlayAnalysisMinDisk13Gb, 13000],
|
||||
];
|
||||
|
||||
/**
|
||||
* The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If
|
||||
@@ -588,24 +597,42 @@ async function checkOverlayAnalysisFeatureEnabled(
|
||||
return new Success(undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the minimum available disk space (in MB) required to perform overlay
|
||||
* analysis, which is the lowest threshold enabled by a feature flag, or the
|
||||
* default threshold if no such feature flag is enabled.
|
||||
*/
|
||||
async function getMinimumDiskSpaceMb(
|
||||
features: FeatureEnablement,
|
||||
): Promise<number> {
|
||||
let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB;
|
||||
for (const [feature, thresholdMb] of OVERLAY_MINIMUM_DISK_SPACE_FEATURES) {
|
||||
if (await features.getValue(feature)) {
|
||||
minimumMb = Math.min(minimumMb, thresholdMb);
|
||||
}
|
||||
}
|
||||
return minimumMb;
|
||||
}
|
||||
|
||||
/** Checks if the runner has enough disk space for overlay analysis. */
|
||||
function runnerHasSufficientDiskSpace(
|
||||
diskUsage: DiskUsage,
|
||||
logger: Logger,
|
||||
useV2ResourceChecks: boolean,
|
||||
minimumDiskSpaceMb: number,
|
||||
): boolean {
|
||||
const minimumDiskSpaceBytes = useV2ResourceChecks
|
||||
? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES
|
||||
: OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES;
|
||||
if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) {
|
||||
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
|
||||
const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000);
|
||||
const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000);
|
||||
if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) {
|
||||
logger.info(
|
||||
`Setting overlay database mode to ${OverlayDatabaseMode.None} ` +
|
||||
`due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` +
|
||||
`minimum of ${minimumDiskSpaceMb} MB.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -637,7 +664,7 @@ async function runnerHasSufficientMemory(
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
|
||||
`Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
@@ -648,12 +675,13 @@ async function runnerHasSufficientMemory(
|
||||
*/
|
||||
async function checkRunnerResources(
|
||||
codeql: CodeQL,
|
||||
features: FeatureEnablement,
|
||||
diskUsage: DiskUsage,
|
||||
ramInput: string | undefined,
|
||||
logger: Logger,
|
||||
useV2ResourceChecks: boolean,
|
||||
): Promise<Result<void, OverlayDisabledReason>> {
|
||||
if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) {
|
||||
const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features);
|
||||
if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) {
|
||||
return new Failure(OverlayDisabledReason.InsufficientDiskSpace);
|
||||
}
|
||||
if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) {
|
||||
@@ -752,9 +780,6 @@ export async function checkOverlayEnablement(
|
||||
Feature.OverlayAnalysisSkipResourceChecks,
|
||||
codeql,
|
||||
));
|
||||
const useV2ResourceChecks = await features.getValue(
|
||||
Feature.OverlayAnalysisResourceChecksV2,
|
||||
);
|
||||
const checkOverlayStatus = await features.getValue(
|
||||
Feature.OverlayAnalysisStatusCheck,
|
||||
);
|
||||
@@ -770,10 +795,10 @@ export async function checkOverlayEnablement(
|
||||
performResourceChecks && diskUsage !== undefined
|
||||
? await checkRunnerResources(
|
||||
codeql,
|
||||
features,
|
||||
diskUsage,
|
||||
ramInput,
|
||||
logger,
|
||||
useV2ResourceChecks,
|
||||
)
|
||||
: new Success<void>(undefined);
|
||||
if (resourceResult.isFailure()) {
|
||||
|
||||
@@ -16,7 +16,7 @@ export interface RemoteFileAddress {
|
||||
}
|
||||
|
||||
/** The default file path to use in configuration file shorthands. */
|
||||
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-action.yaml";
|
||||
export const DEFAULT_CONFIG_FILE_NAME = ".github/codeql-config.yml";
|
||||
|
||||
/** The default ref to use in configuration file shorthands. */
|
||||
export const DEFAULT_CONFIG_FILE_REF = "main";
|
||||
|
||||
+38
-6
@@ -121,12 +121,19 @@ export enum Feature {
|
||||
* `OverlayAnalysisMatchCodeqlVersion` overrides this flag.
|
||||
*/
|
||||
OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run",
|
||||
OverlayAnalysisPython = "overlay_analysis_python",
|
||||
/**
|
||||
* Controls whether lower disk space requirements are used for overlay hardware checks.
|
||||
* Has no effect if `OverlayAnalysisSkipResourceChecks` is enabled.
|
||||
* Feature flags that lower the amount of available disk space that the overlay hardware check
|
||||
* requires. The lowest threshold that is enabled takes effect; if none are enabled, the default
|
||||
* threshold applies. These flags have no effect if `OverlayAnalysisSkipResourceChecks` is
|
||||
* enabled.
|
||||
*/
|
||||
OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2",
|
||||
OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb",
|
||||
OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb",
|
||||
OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb",
|
||||
OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb",
|
||||
OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb",
|
||||
OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb",
|
||||
OverlayAnalysisPython = "overlay_analysis_python",
|
||||
OverlayAnalysisRuby = "overlay_analysis_ruby",
|
||||
/** Controls whether hardware checks are skipped for overlay analysis. */
|
||||
OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks",
|
||||
@@ -354,9 +361,34 @@ export const featureConfig = {
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisResourceChecksV2]: {
|
||||
[Feature.OverlayAnalysisMinDisk8Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2",
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisMinDisk9Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisMinDisk10Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisMinDisk11Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisMinDisk12Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisMinDisk13Gb]: {
|
||||
defaultValue: false,
|
||||
envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB",
|
||||
minimumVersion: undefined,
|
||||
},
|
||||
[Feature.OverlayAnalysisStatusCheck]: {
|
||||
|
||||
+27
-5
@@ -35,6 +35,11 @@ export function isNumber(value: unknown): value is number {
|
||||
return typeof value === "number";
|
||||
}
|
||||
|
||||
/** Asserts that `value` is a boolean. */
|
||||
export function isBoolean(value: unknown): value is boolean {
|
||||
return typeof value === "boolean";
|
||||
}
|
||||
|
||||
/** Asserts that `value` is either a string or undefined. */
|
||||
export function isStringOrUndefined(
|
||||
value: unknown,
|
||||
@@ -62,14 +67,11 @@ function defaultCheck(
|
||||
return (arg) => ({ unknownKeys: [], invalidKeys: [], valid: validate(arg) });
|
||||
}
|
||||
|
||||
function makeValidator<T>(
|
||||
validate: (arg: unknown) => arg is T,
|
||||
required: boolean = true,
|
||||
) {
|
||||
function makeValidator<T>(validate: (arg: unknown) => arg is T) {
|
||||
return {
|
||||
validate,
|
||||
check: defaultCheck(validate),
|
||||
required,
|
||||
required: true,
|
||||
} as const satisfies Validator<T>;
|
||||
}
|
||||
|
||||
@@ -82,6 +84,9 @@ export const string = makeValidator(isString);
|
||||
/** A validator for number fields in schemas. */
|
||||
export const number = makeValidator(isNumber);
|
||||
|
||||
/** A validator for boolean fields in schemas. */
|
||||
export const boolean = makeValidator(isBoolean);
|
||||
|
||||
/** A validator for arrays. */
|
||||
export function array<T>(validator: Validator<T>) {
|
||||
const validate = (val: unknown) => {
|
||||
@@ -221,6 +226,23 @@ export function validateSchema<
|
||||
return result.valid;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that `arr` is an array whose elements satisfy at least `elementSchema`.
|
||||
* Additional keys are accepted in each element.
|
||||
*
|
||||
* @param elementSchema The schema to validate the elements against.
|
||||
* @param arr The array to validate.
|
||||
* @returns Asserts that `arr` has elements of `schema`'s type if validation is successful.
|
||||
*/
|
||||
export function validateArray<
|
||||
S extends Schema,
|
||||
T extends UnvalidatedArray = Array<FromSchema<S>>,
|
||||
>(elementSchema: S, arr: UnvalidatedArray): arr is T {
|
||||
const elementValidator = object(elementSchema);
|
||||
|
||||
return array(elementValidator).validate(arr);
|
||||
}
|
||||
|
||||
export interface CheckSchemaOptions {
|
||||
/** Whether to stop validation after the first error. */
|
||||
failFast?: boolean;
|
||||
|
||||
+1
-7
@@ -83,12 +83,6 @@ export class StartProxyError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
interface StartProxyStatus extends StatusReportBase {
|
||||
// A comma-separated list of registry types which are configured for CodeQL.
|
||||
// This only includes registry types we support, not all that are configured.
|
||||
registry_types: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a status report for the `start-proxy` action indicating a successful outcome.
|
||||
*
|
||||
@@ -112,7 +106,7 @@ export async function sendSuccessStatusReport(
|
||||
logger,
|
||||
);
|
||||
if (statusReportBase !== undefined) {
|
||||
const statusReport: StartProxyStatus = {
|
||||
const statusReport: StatusReportBase = {
|
||||
...statusReportBase,
|
||||
registry_types: registry_types.join(","),
|
||||
};
|
||||
|
||||
@@ -254,13 +254,19 @@ export function credentialToStr(credential: Credential): string {
|
||||
return result;
|
||||
}
|
||||
|
||||
/** A package registry is identified by its type and address. */
|
||||
export type Registry = {
|
||||
/** The schema for `RegistryBase` objects. */
|
||||
export const registryBaseSchema = {
|
||||
/** The type of the package registry. */
|
||||
type: string;
|
||||
type: json.string,
|
||||
/** Whether the registry replaces the base registry for the ecosystem. */
|
||||
"replaces-base"?: boolean;
|
||||
} & Address;
|
||||
"replaces-base": json.optional(json.boolean),
|
||||
} as const satisfies json.Schema;
|
||||
|
||||
/** Information about a registry, other than its address. */
|
||||
export type RegistryBase = json.FromSchema<typeof registryBaseSchema>;
|
||||
|
||||
/** A package registry is identified by its type and address. */
|
||||
export type Registry = RegistryBase & Address;
|
||||
|
||||
// If a registry has an `url`, then that takes precedence over the `host` which may or may
|
||||
// not be defined.
|
||||
|
||||
@@ -4,15 +4,17 @@ import * as uuid from "uuid";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { Config } from "./config-utils";
|
||||
import { EnvVar } from "./environment";
|
||||
import { EnvVar, RegistryProxyVars } from "./environment";
|
||||
import { BuiltInLanguage } from "./languages";
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { ToolsSource } from "./setup-codeql";
|
||||
import type { Registry } from "./start-proxy";
|
||||
import {
|
||||
ActionName,
|
||||
createInitWithConfigStatusReport,
|
||||
createStatusReportBase,
|
||||
getActionsStatus,
|
||||
getRegistryTypesFromEnv,
|
||||
getJobUUID,
|
||||
InitStatusReport,
|
||||
InitWithConfigStatusReport,
|
||||
@@ -22,12 +24,69 @@ import {
|
||||
setupActionsVars,
|
||||
createTestConfig,
|
||||
makeMacro,
|
||||
getTestEnv,
|
||||
RecordingLogger,
|
||||
callee,
|
||||
} from "./testing-utils";
|
||||
import { BuildMode, ConfigurationError, withTmpDir, wrapError } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
test("getRegistryTypesFromEnv - gets unique registry types from environment", async (t) => {
|
||||
const logger = new RecordingLogger(true);
|
||||
const env = getTestEnv({
|
||||
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([
|
||||
{ type: "git_source", url: "https://example.com" },
|
||||
{ type: "git_source", url: "https://github.com" },
|
||||
{ type: "docker_registry", url: "https://registry.example.com" },
|
||||
] satisfies Array<Partial<Registry>>),
|
||||
});
|
||||
|
||||
const result = getRegistryTypesFromEnv(logger, env);
|
||||
t.deepEqual(result, ["git_source", "docker_registry"].sort().join(","));
|
||||
});
|
||||
|
||||
test("getRegistryTypesFromEnv - returns undefined if the env var is not set", async (t) => {
|
||||
const logger = new RecordingLogger(true);
|
||||
const env = getTestEnv({});
|
||||
|
||||
const result = getRegistryTypesFromEnv(logger, env);
|
||||
t.is(result, undefined);
|
||||
});
|
||||
|
||||
test("getRegistryTypesFromEnv - returns undefined if the env var is not valid JSON", async (t) => {
|
||||
const logger = new RecordingLogger(true);
|
||||
const env = getTestEnv({ [RegistryProxyVars.PROXY_URLS]: "[" });
|
||||
|
||||
const result = getRegistryTypesFromEnv(logger, env);
|
||||
t.is(result, undefined);
|
||||
});
|
||||
|
||||
test("getRegistryTypesFromEnv - returns undefined if the env var is unexpected JSON", async (t) => {
|
||||
const logger = new RecordingLogger(true);
|
||||
|
||||
t.is(
|
||||
getRegistryTypesFromEnv(
|
||||
logger,
|
||||
getTestEnv({
|
||||
// Top-level object rather than an array of objects.
|
||||
[RegistryProxyVars.PROXY_URLS]: JSON.stringify({ type: "git_source" }),
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
t.is(
|
||||
getRegistryTypesFromEnv(
|
||||
logger,
|
||||
getTestEnv({
|
||||
// Object has no "type" key.
|
||||
[RegistryProxyVars.PROXY_URLS]: JSON.stringify([{}]),
|
||||
}),
|
||||
),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
test("getJobUUID - generates valid UUIDs", async (t) => {
|
||||
await callee(getJobUUID)
|
||||
.withArgs()
|
||||
@@ -74,6 +133,9 @@ function setupEnvironmentAndStub(tmpDir: string) {
|
||||
|
||||
process.env[EnvVar.ANALYSIS_KEY] = "analysis-key";
|
||||
process.env["ImageVersion"] = "2023.05.19.1";
|
||||
process.env[RegistryProxyVars.PROXY_URLS] = JSON.stringify([
|
||||
{ type: "maven_repository" },
|
||||
] satisfies Array<Partial<Registry>>);
|
||||
|
||||
const getRequiredInput = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
getRequiredInput.withArgs("matrix").resolves("input/matrix");
|
||||
@@ -117,6 +179,7 @@ test.serial("createStatusReportBase", async (t) => {
|
||||
t.is(typeof statusReport.job_run_uuid, "string");
|
||||
t.is(statusReport.languages, "java,swift");
|
||||
t.is(statusReport.ref, process.env["GITHUB_REF"]!);
|
||||
t.is(statusReport.registry_types, "maven_repository");
|
||||
t.is(statusReport.runner_available_disk_space_bytes, 100);
|
||||
t.is(statusReport.runner_image_version, process.env["ImageVersion"]);
|
||||
t.is(statusReport.runner_os, process.env["RUNNER_OS"]!);
|
||||
|
||||
+54
-1
@@ -19,12 +19,14 @@ import type { ComputedInput, InputName } from "./config/inputs";
|
||||
import { parseRegistriesWithoutCredentials } from "./config/pack-registries";
|
||||
import type { DependencyCacheRestoreStatusReport } from "./dependency-caching";
|
||||
import { DocUrl } from "./doc-url";
|
||||
import { EnvVar } from "./environment";
|
||||
import { EnvVar, getEnv, ReadOnlyEnv, RegistryProxyVars } from "./environment";
|
||||
import { getRef } from "./git-utils";
|
||||
import * as json from "./json";
|
||||
import type { Logger } from "./logging";
|
||||
import type { OverlayBaseDatabaseDownloadStats } from "./overlay/caching";
|
||||
import { getRepositoryNwo } from "./repository";
|
||||
import type { ToolsSource } from "./setup-codeql";
|
||||
import { registryBaseSchema } from "./start-proxy/types";
|
||||
import {
|
||||
ConfigurationError,
|
||||
getRequiredEnvParam,
|
||||
@@ -185,6 +187,12 @@ export interface StatusReportBase {
|
||||
ml_powered_javascript_queries?: string;
|
||||
/** Ref that the workflow was triggered on. */
|
||||
ref: string;
|
||||
/**
|
||||
* A comma-separated list of private registry types which are configured for CodeQL.
|
||||
* This only includes registry types we support (as determined by the `start-proxy` action),
|
||||
* not all that are configured.
|
||||
*/
|
||||
registry_types?: string;
|
||||
/** Action runner hardware architecture (context runner.arch). */
|
||||
runner_arch?: string;
|
||||
/** Available disk space on the runner, in bytes. */
|
||||
@@ -288,6 +296,50 @@ export interface EventReport {
|
||||
started_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to retrieve a list of private registry types from the `CODEQL_PROXY_URLS` environment
|
||||
* variable and returns it as a comma-separated string if successful. Returns `undefined` otherwise.
|
||||
*/
|
||||
export function getRegistryTypesFromEnv(
|
||||
logger: Logger,
|
||||
env: ReadOnlyEnv = getEnv(),
|
||||
): string | undefined {
|
||||
// Try to get the value of the environment variable.
|
||||
const value = env.getOptional(RegistryProxyVars.PROXY_URLS);
|
||||
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Try to parse the JSON we expect to find in it and return the comma-separated list of
|
||||
// (unique) registry types.
|
||||
try {
|
||||
const data = JSON.parse(value) as unknown;
|
||||
|
||||
// Check that the parsed JSON meets our expectations.
|
||||
if (!json.isArray(data)) {
|
||||
logger.debug(
|
||||
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array, but got '${typeof data}'.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (!json.validateArray(registryBaseSchema, data)) {
|
||||
logger.debug(
|
||||
`Expected '${RegistryProxyVars.PROXY_URLS}' to contain a JSON array of registry objects, but got something else.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const types = new Set(data.map((r) => r.type));
|
||||
return Array.from(types).sort().join(",");
|
||||
} catch (err) {
|
||||
logger.debug(
|
||||
`Failed to parse '${RegistryProxyVars.PROXY_URLS}': ${getErrorMessage(err)}.`,
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose a StatusReport.
|
||||
*
|
||||
@@ -350,6 +402,7 @@ export async function createStatusReportBase(
|
||||
job_name: jobName,
|
||||
job_run_uuid: jobRunUUID,
|
||||
ref,
|
||||
registry_types: getRegistryTypesFromEnv(logger),
|
||||
runner_os: runnerOs,
|
||||
started_at: workflowStartedAt,
|
||||
status,
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import * as path from "path";
|
||||
import * as stream from "stream";
|
||||
|
||||
import test from "ava";
|
||||
|
||||
import { getRunnerLogger } from "./logging";
|
||||
import { extractTarZst } from "./tar";
|
||||
import { setupTests } from "./testing-utils";
|
||||
import { withTmpDir } from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
test("extractTarZst rejects if the input stream errors", async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
const archive = new stream.PassThrough();
|
||||
const promise = extractTarZst(
|
||||
archive,
|
||||
path.join(tmpDir, "dest"),
|
||||
{ type: "gnu", version: "1.34" },
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
archive.destroy(
|
||||
Object.assign(new Error("socket hang up"), {
|
||||
code: "ECONNRESET",
|
||||
}),
|
||||
);
|
||||
|
||||
await t.throwsAsync(promise, {
|
||||
message: /Error while downloading and extracting tar/,
|
||||
});
|
||||
});
|
||||
});
|
||||
+9
-4
@@ -194,10 +194,15 @@ export async function extractTarZst(
|
||||
});
|
||||
|
||||
if (tar instanceof stream.Readable) {
|
||||
tar.pipe(tarProcess.stdin).on("error", (err) => {
|
||||
reject(
|
||||
new Error(`Error while downloading and extracting tar: ${err}`),
|
||||
);
|
||||
// Use `pipeline` rather than `pipe` so that an error on either stream is reported here
|
||||
// rather than being emitted as an unhandled `error` event, and so that `tar`'s standard
|
||||
// input is closed if the download fails partway through.
|
||||
stream.pipeline(tar, tarProcess.stdin, (err) => {
|
||||
if (err) {
|
||||
reject(
|
||||
new Error(`Error while downloading and extracting tar: ${err}`),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -38,6 +38,43 @@ test.serial(
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract falls back to downloading before extracting if streaming fails",
|
||||
async (t) => {
|
||||
await withTmpDir(async (tmpDir) => {
|
||||
sinon.stub(process, "platform").value("linux");
|
||||
const archivePath = path.join(tmpDir, "codeql-bundle.tar.zst");
|
||||
const destination = path.join(tmpDir, "codeql");
|
||||
const downloadTool = sinon
|
||||
.stub(toolcache, "downloadTool")
|
||||
.resolves(archivePath);
|
||||
const extract = sinon.stub(tar, "extract").resolves(destination);
|
||||
const extractTarZst = sinon.stub(tar, "extractTarZst").resolves();
|
||||
const request = nock("https://example.com")
|
||||
.get("/codeql-bundle.tar.zst")
|
||||
.replyWithError(
|
||||
Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }),
|
||||
);
|
||||
|
||||
const statusReport = await downloadAndExtract(
|
||||
"https://example.com/codeql-bundle.tar.zst",
|
||||
"zstd",
|
||||
destination,
|
||||
undefined,
|
||||
{},
|
||||
{ type: "gnu", version: "1.34" },
|
||||
getRunnerLogger(true),
|
||||
);
|
||||
|
||||
t.assert(Number.isInteger(statusReport.downloadDurationMs));
|
||||
t.true(request.isDone());
|
||||
t.false(extractTarZst.called);
|
||||
t.true(downloadTool.calledOnce);
|
||||
t.true(extract.calledOnce);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test.serial(
|
||||
"downloadAndExtract omits the download duration when streaming extraction",
|
||||
async (t) => {
|
||||
|
||||
+24
-4
@@ -19,6 +19,12 @@ import { cleanUpPath, getErrorMessage, getRequiredEnvParam } from "./util";
|
||||
*/
|
||||
const STREAMING_HIGH_WATERMARK_BYTES = 4 * 1024 * 1024; // 4 MiB
|
||||
|
||||
/**
|
||||
* How long the streaming download of the CodeQL tools may stall for before we abort it. This
|
||||
* applies both to establishing the connection and to gaps between chunks of the response body.
|
||||
*/
|
||||
const STREAMING_STALL_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
/**
|
||||
* The name of the tool cache directory for the CodeQL tools.
|
||||
*/
|
||||
@@ -137,8 +143,8 @@ async function downloadAndExtractZstdWithStreaming(
|
||||
authorization ? { authorization } : {},
|
||||
headers,
|
||||
);
|
||||
const response = await new Promise<IncomingMessage>((resolve) =>
|
||||
https.get(
|
||||
const response = await new Promise<IncomingMessage>((resolve, reject) => {
|
||||
const request = https.get(
|
||||
codeqlURL,
|
||||
{
|
||||
headers,
|
||||
@@ -148,10 +154,24 @@ async function downloadAndExtractZstdWithStreaming(
|
||||
agent,
|
||||
} as unknown as RequestOptions,
|
||||
(r) => resolve(r),
|
||||
),
|
||||
);
|
||||
);
|
||||
// Without this listener, connection failures such as `ECONNRESET` are emitted as unhandled
|
||||
// `error` events, which terminate the process instead of letting us fall back to downloading
|
||||
// the bundle before extracting it. This listener stays attached after the response arrives, so
|
||||
// it also handles errors that occur while the response is being streamed.
|
||||
request.on("error", reject);
|
||||
request.setTimeout(STREAMING_STALL_TIMEOUT_MS, () => {
|
||||
request.destroy(
|
||||
new Error(
|
||||
`No data received for ${formatDuration(STREAMING_STALL_TIMEOUT_MS)}.`,
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
if (response.statusCode !== 200) {
|
||||
// Discard the response body so that the connection can be released.
|
||||
response.resume();
|
||||
throw new Error(
|
||||
`Failed to download CodeQL bundle from ${codeqlURL}. HTTP status code: ${response.statusCode}.`,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user