Refactor CLI caching with in-memory and file storage

This commit is contained in:
Mario Campos
2026-06-17 23:45:01 -05:00
parent dc8e1e9aa0
commit a602287f73
8 changed files with 432 additions and 262 deletions
+226
View File
@@ -0,0 +1,226 @@
import * as fs from "fs";
import * as os from "os";
import path from "path";
import test from "ava";
import {
cacheCommandOutput,
getCachedCommandOutput,
resetCachedCommandOutputs,
CommandCacheKey,
} from "./cache";
import { isVersionInfo } from "./codeql";
import { setupTests } from "./testing-utils";
setupTests(test);
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
/**
* Runs `body` with a temporary directory configured as the cache's backing
* store (`RUNNER_TEMP`). `CODEQL_ACTION_TEMP` is cleared so that
* `getTemporaryDirectory()` falls back to `RUNNER_TEMP`.
*
* `setupTests` snapshots and restores `process.env` around every test, so we
* don't restore the environment variables we set here ourselves.
*/
async function withCacheDir(
body: (cacheFilePath: string) => Promise<void> | void,
): Promise<void> {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "cache-test-"));
process.env["RUNNER_TEMP"] = tmpDir;
delete process.env["CODEQL_ACTION_TEMP"];
resetCachedCommandOutputs();
try {
await body(path.join(tmpDir, COMMAND_CACHE_FILENAME));
} finally {
await fs.promises.rm(tmpDir, { force: true, recursive: true });
}
}
function writeCacheFile(
cacheFilePath: string,
contents: Record<string, unknown>,
): void {
fs.writeFileSync(cacheFilePath, JSON.stringify(contents));
}
test.serial(
"getCachedCommandOutput reuses an output persisted by an earlier step",
async (t) => {
await withCacheDir((cacheFilePath) => {
writeCacheFile(cacheFilePath, {
[CommandCacheKey.Version]: {
cmd: "/path/to/codeql",
output: { version: "2.20.0" },
},
});
t.deepEqual(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
{ version: "2.20.0" },
);
});
},
);
test.serial(
"getCachedCommandOutput ignores an output persisted from a different CLI",
async (t) => {
await withCacheDir((cacheFilePath) => {
writeCacheFile(cacheFilePath, {
[CommandCacheKey.Version]: {
cmd: "/path/to/other-codeql",
output: { version: "2.20.0" },
},
});
t.is(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
undefined,
);
});
},
);
test.serial(
"getCachedCommandOutput ignores a malformed cache file",
async (t) => {
await withCacheDir((cacheFilePath) => {
fs.writeFileSync(cacheFilePath, "not valid json");
t.is(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
undefined,
);
});
},
);
test.serial(
"getCachedCommandOutput returns undefined when there is no cache file",
async (t) => {
await withCacheDir(() => {
t.is(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
undefined,
);
});
},
);
test.serial(
"getCachedCommandOutput ignores an output that fails validation",
async (t) => {
await withCacheDir((cacheFilePath) => {
for (const output of [
{},
{ version: 2 },
{ version: "2.20.0", overlayVersion: "1" },
{ version: "2.20.0", features: "nope" },
]) {
resetCachedCommandOutputs();
writeCacheFile(cacheFilePath, {
[CommandCacheKey.Version]: { cmd: "/path/to/codeql", output },
});
t.is(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
undefined,
JSON.stringify(output),
);
}
});
},
);
test.serial(
"getCachedCommandOutput ignores an entry missing the cmd field",
async (t) => {
await withCacheDir((cacheFilePath) => {
writeCacheFile(cacheFilePath, {
[CommandCacheKey.Version]: { output: { version: "2.20.0" } },
});
t.is(
getCachedCommandOutput(
CommandCacheKey.Version,
"/path/to/codeql",
isVersionInfo,
),
undefined,
);
});
},
);
test.serial(
"cacheCommandOutput persists the output to both the memo and the file",
async (t) => {
await withCacheDir((cacheFilePath) => {
cacheCommandOutput("some-command", "/path/to/codeql", {
hello: "world",
});
// Tier 2: the temporary file contains the entry.
const onDisk = JSON.parse(
fs.readFileSync(cacheFilePath, "utf8"),
) as Record<string, unknown>;
t.deepEqual(onDisk["some-command"], {
cmd: "/path/to/codeql",
output: { hello: "world" },
});
// Tier 1: the value is served from the memo even after the file is gone.
fs.rmSync(cacheFilePath);
t.deepEqual(getCachedCommandOutput("some-command", "/path/to/codeql"), {
hello: "world",
});
});
},
);
test.serial(
"getCachedCommandOutput prefers the in-memory memo over the file",
async (t) => {
await withCacheDir((cacheFilePath) => {
cacheCommandOutput("some-command", "/path/to/codeql", { value: 1 });
// Overwrite the file with a different value; the memo (tier 1) should win.
writeCacheFile(cacheFilePath, {
"some-command": { cmd: "/path/to/codeql", output: { value: 2 } },
});
t.deepEqual(getCachedCommandOutput("some-command", "/path/to/codeql"), {
value: 1,
});
});
},
);
test.serial(
"cacheCommandOutput throws if called twice for the same key",
async (t) => {
await withCacheDir(() => {
cacheCommandOutput("some-command", "/path/to/codeql", { value: 1 });
t.throws(() =>
cacheCommandOutput("some-command", "/path/to/codeql", { value: 2 }),
);
});
},
);
+147
View File
@@ -0,0 +1,147 @@
import * as fs from "fs";
import * as path from "path";
import { getTemporaryDirectory } from "./actions-util";
import * as json from "./json";
/** The name of the temporary file backing the cache (tier 2). */
const COMMAND_CACHE_FILENAME = "codeql-action-command-cache.json";
/**
* The keys under which the output of cached `codeql` commands is stored. Each
* key is shared by the producer (the corresponding method in `codeql.ts`) and
* any consumers (e.g. `status-report.ts`, which peeks the cached version
* without invoking the CLI).
*/
export enum CommandCacheKey {
Version = "version",
ResolveLanguages = "resolveLanguages",
}
/** A single cached command output together with the CLI path it came from. */
interface CacheEntry {
/**
* The path to the CodeQL CLI that produced `output`. Persisted so that a
* different step using a different CodeQL bundle doesn't pick up a stale
* value.
*/
cmd: string;
output: unknown;
}
/**
* Tier 1: the in-process memo. Consulted first on every lookup and populated
* whenever a value is read from the file (tier 2) or computed via the CLI
* (tier 3).
*/
const inMemoryCache = new Map<string, CacheEntry>();
function getCommandCacheFilePath(): string {
return path.join(getTemporaryDirectory(), COMMAND_CACHE_FILENAME);
}
/**
* Reads and parses the temporary cache file. Best-effort: a missing, malformed,
* or otherwise unreadable file is treated as an empty cache.
*/
function readCommandCacheFile(): Record<string, CacheEntry> {
let contents: string;
try {
contents = fs.readFileSync(getCommandCacheFilePath(), "utf8");
} catch {
return {};
}
try {
const parsed = json.parseString(contents);
if (json.isObject(parsed)) {
return parsed;
}
} catch {
// Fall through and treat a malformed file as empty.
}
return {};
}
/**
* Persists the cache to the temporary file. Best-effort: a failure to write
* just means a later step re-runs the CLI.
*/
function writeCommandCacheFile(data: Record<string, CacheEntry>): void {
try {
fs.writeFileSync(getCommandCacheFilePath(), JSON.stringify(data));
} catch {
// Best-effort; ignore write failures.
}
}
/**
* Stores the output of a command under `key`, writing it to both the in-memory
* memo (tier 1) and the temporary file (tier 2).
*
* Should only be called once per key within a single process; doing otherwise
* indicates a logic error, since a value that has already been cached should be
* served from the memo rather than recomputed.
*/
export function cacheCommandOutput(
key: string,
cmd: string,
output: unknown,
): void {
if (inMemoryCache.has(key)) {
throw new Error(
`cacheCommandOutput() should be called only once per key, but was called more than once for '${key}'.`,
);
}
const entry: CacheEntry = { cmd, output };
inMemoryCache.set(key, entry);
const data = readCommandCacheFile();
data[key] = entry;
writeCommandCacheFile(data);
}
/**
* Returns the cached output for `key`, or `undefined` if it isn't cached.
*
* Resolves tier 1 (in-memory memo) first, then tier 2 (temporary file). A value
* loaded from the file is ignored unless its `cmd` matches the optional `cmd`
* argument, and it satisfies the optional `validate` type guard; valid values
* are memoized into tier 1 before being returned.
*
* A return value of `undefined` signals the caller to fall back to tier 3 (the
* CLI).
*/
export function getCachedCommandOutput<T>(
key: string,
cmd?: string,
validate?: (output: unknown) => output is T,
): T | undefined {
// Tier 1: the in-memory variable.
const memoized = inMemoryCache.get(key);
if (memoized !== undefined) {
return memoized.output as T;
}
// Tier 2: the temporary file persisted by an earlier step, if any.
const entry = readCommandCacheFile()[key] as unknown;
if (
!json.isObject<CacheEntry>(entry) ||
!json.isString(entry.cmd) ||
(cmd !== undefined && entry.cmd !== cmd) ||
(validate !== undefined && !validate(entry.output))
) {
return undefined;
}
// Memoize so subsequent lookups in this process hit tier 1.
inMemoryCache.set(key, { cmd: entry.cmd, output: entry.output });
return entry.output as T;
}
/**
* Clears the in-process memo (tier 1). Only for use in tests, which exercise
* multiple "steps" within a single process.
*/
export function resetCachedCommandOutputs(): void {
inMemoryCache.clear();
}
+47 -35
View File
@@ -12,6 +12,11 @@ import {
runTool,
} from "./actions-util";
import * as api from "./api-client";
import {
cacheCommandOutput,
getCachedCommandOutput,
CommandCacheKey,
} from "./cache";
import { CliError, wrapCliConfigurationError } from "./cli-errors";
import { appendExtraQueryExclusions, type Config } from "./config-utils";
import { DocUrl } from "./doc-url";
@@ -22,6 +27,7 @@ import {
FeatureEnablement,
} from "./feature-flags";
import { isAnalyzingDefaultBranch } from "./git-utils";
import * as json from "./json";
import { Language } from "./languages";
import { Logger } from "./logging";
import { writeBaseDatabaseOidsFile, writeOverlayChangesFile } from "./overlay";
@@ -230,6 +236,20 @@ export interface VersionInfo {
overlayVersion?: number;
}
export function isVersionInfo(x: unknown): x is VersionInfo {
return (
json.isObject(x) &&
json.validateSchema(
{
version: json.string,
features: json.undefinable(json.object),
overlayVersion: json.undefinable(json.number),
},
x,
)
);
}
export interface ResolveDatabaseOutput {
overlayBaseSpecifier?: string;
}
@@ -246,6 +266,15 @@ export interface ResolveLanguagesOutput {
};
}
export function isResolveLanguagesOutput(
x: unknown,
): x is ResolveLanguagesOutput {
return (
json.isObject(x)
// TODO: finish this with Copilot
);
}
export interface ResolveBuildEnvironmentOutput {
configuration?: {
[language: string]: {
@@ -723,41 +752,24 @@ async function getCodeQLForCmd(
];
await runCli(cmd, args);
},
async resolveLanguages(
{
filterToLanguagesWithQueries,
}: {
filterToLanguagesWithQueries: boolean;
} = { filterToLanguagesWithQueries: false },
) {
async function runCliResolveLanguages() {
const codeqlArgs = [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...(filterToLanguagesWithQueries
? ["--filter-to-languages-with-queries"]
: []),
...getExtraOptionsFromEnv(["resolve", "languages"]),
];
const output = await runCli(cmd, codeqlArgs);
try {
return JSON.parse(output) as ResolveLanguagesOutput;
} catch (e) {
throw new Error(
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`,
);
}
}
let result = util.getCachedCodeQlResolveLanguages(cmd);
if (result === undefined) {
result = await runCliResolveLanguages();
util.cacheCodeQlResolveLanguages(cmd, result);
}
return result;
async resolveLanguages() {
return getCachedOrRun(
CommandCacheKey.ResolveLanguages,
cmd,
() =>
runCliJson<ResolveLanguagesOutput>(cmd, [
"resolve",
"languages",
"--format=betterjson",
"--extractor-options-verbosity=4",
"--extractor-include-aliases",
...(filterToLanguagesWithQueries
? ["--filter-to-languages-with-queries"]
: []),
...getExtraOptionsFromEnv(["resolve", "languages"]),
]),
isResolveLanguagesOutput,
);
},
async resolveBuildEnvironment(
workingDir: string | undefined,
-12
View File
@@ -17,18 +17,6 @@ export enum EnvVar {
*/
CLI_VERBOSITY = "CODEQL_VERBOSITY",
/**
* `PersistedVersionInfo` for the CodeQL CLI, so later Actions steps can reuse it instead of
* invoking `codeql version` again.
*/
CODEQL_VERSION_INFO = "CODEQL_ACTION_CLI_VERSION_INFO",
/**
* `ResolveLanguagesOutput` for the CodeQL CLI, so later Actions steps can reuse it instead of
* invoking `codeql resolve languages` again.
*/
CODEQL_RESOLVE_LANGUAGES = "CODEQL_ACTION_CLI_RESOLVE_LANGUAGES",
/** Whether the CodeQL Action has invoked the Go autobuilder. */
DID_AUTOBUILD_GOLANG = "CODEQL_ACTION_DID_AUTOBUILD_GOLANG",
+7 -2
View File
@@ -12,6 +12,8 @@ import {
isSelfHostedRunner,
} from "./actions-util";
import { getAnalysisKey, getApiClient } from "./api-client";
import { getCachedCommandOutput, CommandCacheKey } from "./cache";
import { isVersionInfo } from "./codeql";
import { parseRegistriesWithoutCredentials, type Config } from "./config-utils";
import { DependencyCacheRestoreStatusReport } from "./dependency-caching";
import { DocUrl } from "./doc-url";
@@ -24,7 +26,6 @@ import { ToolsSource } from "./setup-codeql";
import {
ConfigurationError,
getRequiredEnvParam,
getCachedCodeQlVersion,
isInTestMode,
GITHUB_DOTCOM_URL,
DiskUsage,
@@ -283,7 +284,11 @@ export async function createStatusReportBase(
core.exportVariable(EnvVar.WORKFLOW_STARTED_AT, workflowStartedAt);
}
const runnerOs = getRequiredEnvParam("RUNNER_OS");
const codeQlCliVersion = getCachedCodeQlVersion();
const codeQlCliVersion = getCachedCommandOutput(
CommandCacheKey.Version,
undefined,
isVersionInfo,
);
const actionRef = process.env["GITHUB_ACTION_REF"] || "";
const testingEnvironment = getTestingEnvironment();
// re-export the testing environment variable so that it is available to subsequent steps,
+4 -4
View File
@@ -14,6 +14,7 @@ import { ActionsEnv, getActionVersion } from "./actions-util";
import { AnalysisKind } from "./analyses";
import * as apiClient from "./api-client";
import { GitHubApiDetails } from "./api-client";
import { resetCachedCommandOutputs } from "./cache";
import { CachingKind } from "./caching-utils";
import * as codeql from "./codeql";
import { Config } from "./config-utils";
@@ -32,7 +33,6 @@ import {
GitHubVariant,
GitHubVersion,
HTTPError,
resetCachedCodeQlVersion,
} from "./util";
export const SAMPLE_DOTCOM_API_DETAILS = {
@@ -102,9 +102,9 @@ export function setupTests(testFn: TestFn<any>) {
// unless the test explicitly sets one up.
codeql.setCodeQL({});
// Reset the in-process CodeQL version cache so that it doesn't leak between
// tests, which each represent a separate Actions step in production.
resetCachedCodeQlVersion();
// Reset the in-process CodeQL command-output cache so that it doesn't leak
// between tests, which each represent a separate Actions step in production.
resetCachedCommandOutputs();
// Replace stdout and stderr so we can record output during tests
t.context.testOutput = "";
-55
View File
@@ -532,58 +532,3 @@ test("Failure.orElse returns the default value for a failure result", (t) => {
const result = new util.Failure(new Error("test error"));
t.is(result.orElse("default value"), "default value");
});
test.serial(
"getCachedCodeQlVersion reuses a version persisted by an earlier step",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0" },
});
t.deepEqual(util.getCachedCodeQlVersion("/path/to/codeql"), {
version: "2.20.0",
});
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted version from a different CLI",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = JSON.stringify({
cmd: "/path/to/other-codeql",
version: { version: "2.20.0" },
});
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a malformed persisted value",
(t) => {
process.env[EnvVar.CODEQL_VERSION_INFO] = "not valid json";
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined);
},
);
test.serial(
"getCachedCodeQlVersion ignores a persisted value with the wrong structure",
(t) => {
for (const value of [
JSON.stringify({ cmd: "/path/to/codeql" }),
JSON.stringify({ cmd: "/path/to/codeql", version: {} }),
JSON.stringify({ cmd: "/path/to/codeql", version: { version: 2 } }),
JSON.stringify({ version: { version: "2.20.0" } }),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", overlayVersion: "1" },
}),
JSON.stringify({
cmd: "/path/to/codeql",
version: { version: "2.20.0", features: "nope" },
}),
]) {
process.env[EnvVar.CODEQL_VERSION_INFO] = value;
t.is(util.getCachedCodeQlVersion("/path/to/codeql"), undefined, value);
}
},
);
+1 -154
View File
@@ -10,7 +10,7 @@ import * as yaml from "js-yaml";
import * as semver from "semver";
import * as apiCompatibility from "./api-compatibility.json";
import type { CodeQL, VersionInfo, ResolveLanguagesOutput } from "./codeql";
import type { CodeQL } from "./codeql";
import type { Pack } from "./config/db-config";
import type { Config } from "./config-utils";
import { EnvVar } from "./environment";
@@ -617,159 +617,6 @@ export function asHTTPError(arg: any): HTTPError | undefined {
return undefined;
}
let cachedCodeQlVersion: undefined | VersionInfo = undefined;
/**
* Resets the in-process cache of the CodeQL CLI version. Only for use in tests,
* which exercise multiple "steps" within a single process.
*/
export function resetCachedCodeQlVersion(): void {
cachedCodeQlVersion = undefined;
}
/** The persisted version together with the CLI path it was obtained from. */
interface PersistedVersionInfo {
cmd: string;
version: VersionInfo;
}
function isVersionInfo(x: unknown): x is VersionInfo {
return (
json.isObject(x) &&
json.validateSchema(
{
version: json.string,
features: json.undefinable(json.object),
overlayVersion: json.undefinable(json.number),
},
x,
)
);
}
function isPersistedVersionInfo(x: unknown): x is PersistedVersionInfo {
const candidate = x as Partial<PersistedVersionInfo> | null;
return (
typeof candidate === "object" &&
candidate !== null &&
typeof candidate.cmd === "string" &&
isVersionInfo(candidate.version)
);
}
export function cacheCodeQlVersion(cmd: string, version: VersionInfo): void {
if (cachedCodeQlVersion !== undefined) {
throw new Error("cacheCodeQlVersion() should be called only once");
}
cachedCodeQlVersion = version;
// Persist the version so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql version` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale version.
core.exportVariable(
EnvVar.CODEQL_VERSION_INFO,
JSON.stringify({ cmd, version }),
);
}
export function getCachedCodeQlVersion(cmd?: string): undefined | VersionInfo {
if (cachedCodeQlVersion !== undefined) {
return cachedCodeQlVersion;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql version` instead.
const serialized = process.env[EnvVar.CODEQL_VERSION_INFO];
if (!serialized) {
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch {
return undefined;
}
if (
!isPersistedVersionInfo(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlVersion = persisted.version;
return cachedCodeQlVersion;
}
let cachedCodeQlResolveLanguages: undefined | ResolveLanguagesOutput =
undefined;
interface PersistedResolveLanguagesOutput {
cmd: string;
output: ResolveLanguagesOutput;
}
export function cacheCodeQlResolveLanguages(
cmd: string,
output: ResolveLanguagesOutput,
): void {
if (cachedCodeQlResolveLanguages !== undefined) {
throw new Error("cacheCodeQlResolveLanguages() should be called only once");
}
cachedCodeQlResolveLanguages = output;
// Persist the output so that subsequent Actions steps, which run in separate
// processes, can reuse it rather than invoking `codeql resolve languages` again. We
// record the CLI path so that a different step using a different CodeQL bundle
// doesn't pick up a stale output.
core.exportVariable(
EnvVar.CODEQL_RESOLVE_LANGUAGES,
JSON.stringify({ cmd, output }),
);
}
function isPersistedResolveLanguagesOutput(
value: unknown,
): value is PersistedResolveLanguagesOutput {
return (
typeof value === "object" &&
value !== null &&
typeof (value as Record<string, unknown>).cmd === "string" &&
typeof (value as Record<string, unknown>).output === "object" &&
(value as Record<string, unknown>).output !== null
);
}
export function getCachedCodeQlResolveLanguages(
cmd?: string,
): undefined | ResolveLanguagesOutput {
if (cachedCodeQlResolveLanguages !== undefined) {
return cachedCodeQlResolveLanguages;
}
// Fall back to the value persisted by an earlier Actions step, if any. This is
// best-effort: any malformed or mismatched value is ignored so that the caller
// invokes `codeql resolve languages` instead.
const serialized = process.env[EnvVar.CODEQL_RESOLVE_LANGUAGES];
if (!serialized) {
return undefined;
}
let persisted: unknown;
try {
persisted = JSON.parse(serialized);
} catch {
return undefined;
}
if (
!isPersistedResolveLanguagesOutput(persisted) ||
(cmd !== undefined && persisted.cmd !== cmd)
) {
return undefined;
}
// Memoize the parsed value so that subsequent calls in this process don't
// re-parse the environment variable.
cachedCodeQlResolveLanguages = persisted.output;
return cachedCodeQlResolveLanguages;
}
export async function codeQlVersionAtLeast(
codeql: CodeQL,
requiredVersion: string,