Merge pull request #1527 from github/aeisenberg/qlconfig-in-cli

Ensure qlconfig file is created when config parsing in cli is on

.github/workflows/__init-with-registries.yml

@@ -25,6 +25,18 @@ jobs:
     strategy:
       matrix:
         include:
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: windows-latest
+          version: cached
+        - os: ubuntu-latest
+          version: latest
+        - os: macos-latest
+          version: latest
+        - os: windows-latest
+          version: latest
         - os: ubuntu-latest
           version: nightly-latest
         - os: macos-latest
@@ -75,5 +87,35 @@ jobs:
             echo "::error $CODEQL_PACK1 pack was not installed."
             exit 1
         fi
+
+    - name: Verify qlconfig.yml file was created
+      shell: bash
+      run: |
+        QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
+        echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
+        if [[ -f $QLCONFIG_PATH ]]
+        then
+            echo "qlconfig.yml file was created."
+        else
+            echo "::error qlconfig.yml file was not created."
+            exit 1
+        fi
+
+    - name: Verify contents of qlconfig.yml
+    # yq is not available on windows
+      if: runner.os != 'Windows'
+      shell: bash
+      run: |
+        QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
+        cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
+        if [[ $? -eq 0 ]]
+        then
+            echo "Registry was added to qlconfig.yml file."
+        else
+            echo "::error Registry was not added to qlconfig.yml file."
+            echo "Contents of qlconfig.yml file:"
+            cat $QLCONFIG_PATH
+            exit 1
+        fi
     env:
       CODEQL_ACTION_TEST_MODE: true

.github/workflows/script/check-node-modules.sh

@@ -7,17 +7,9 @@ if [ ! -z "$(git status --porcelain)" ]; then
     >&2 echo "Failed: Repo should be clean before testing!"
     exit 1
 fi
-# When updating this, make sure to update the npm version in
-# `.github/workflows/update-dependencies.yml` too.
-sudo npm install --force -g npm@9.2.0
 
-# clean the npm cache to ensure we don't have any files owned by root
+"$(dirname "$0")/update-node-modules.sh" check-only
-sudo npm cache clean --force
 
-# Reinstall modules and then clean to remove absolute paths
-# Use 'npm ci' instead of 'npm install' as this is intended to be reproducible
-npm ci
-npm run removeNPMAbsolutePaths
 # Check that repo is still clean
 if [ ! -z "$(git status --porcelain)" ]; then
     # If we get a fail here then the PR needs attention

.github/workflows/script/update-node-modules.sh

@@ -0,0 +1,18 @@
+if [ "$1" != "update" && "$1" != "check-only" ]; then
+    >&2 echo "Failed: Invalid argument. Must be 'update' or 'check-only'"
+    exit 1
+fi
+
+sudo npm install --force -g npm@9.2.0
+
+# clean the npm cache to ensure we don't have any files owned by root
+sudo npm cache clean --force
+
+if [ "$1" = "update" ]; then
+    npm install
+fi
+
+# Reinstall modules and then clean to remove absolute paths
+# Use 'npm ci' instead of 'npm install' as this is intended to be reproducible
+npm ci
+npm run removeNPMAbsolutePaths

.github/workflows/update-dependencies.yml

@@ -27,12 +27,7 @@ jobs:
       run: |
         git fetch origin "$BRANCH" --depth=1
         git checkout "origin/$BRANCH"
-        # When updating this, make sure to update the npm version in
+        .github/workflows/script/update-node-modules.sh update
-        # `.github/workflows/script/check-node-modules.sh` too.
-        sudo npm install --force -g npm@9.2.0
-        npm install
-        npm ci
-        npm run removeNPMAbsolutePaths
         if [ ! -z "$(git status --porcelain)" ]; then
           git config --global user.email "github-actions@github.com"
           git config --global user.name "github-actions[bot]"

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CodeQL Action Changelog
 
+## [UNRELEASED]
+
+No user facing changes.
+
 ## 2.2.5 - 24 Feb 2023
 
 - Update default CodeQL bundle version to 2.12.3. [#1543](https://github.com/github/codeql-action/pull/1543)

lib/codeql.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
+exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_TRACING_GLIBC_2_34 = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -98,6 +98,10 @@ exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
  * Versions 2.11.1+ of the CodeQL Bundle include a `security-experimental` built-in query suite for each language.
  */
 exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
+/**
+ * Versions 2.12.4+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
+ */
+exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
 /**
  * Set up CodeQL CLI access.
  *
@@ -302,7 +306,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 ...getExtraOptionsFromEnv(["database", "init"]),
             ]);
         },
-        async databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger) {
+        async databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger) {
             const extraArgs = config.languages.map((language) => `--language=${language}`);
             if (config.languages.filter((l) => (0, languages_1.isTracedLanguage)(l)).length > 0) {
                 extraArgs.push("--begin-tracing");
@@ -320,17 +324,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                     extraArgs.push("--no-internal-use-lua-tracing");
                 }
             }
-            // A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
+            // A code scanning config file is only generated if the CliConfigFileEnabled feature flag is enabled.
-            const configLocation = await generateCodeScanningConfig(codeql, config, featureEnablement, logger);
+            const codeScanningConfigFile = await generateCodeScanningConfig(codeql, config, featureEnablement, logger);
             // Only pass external repository token if a config file is going to be parsed by the CLI.
             let externalRepositoryToken;
-            if (configLocation) {
+            if (codeScanningConfigFile) {
-                extraArgs.push(`--codescanning-config=${configLocation}`);
                 externalRepositoryToken = (0, actions_util_1.getOptionalInput)("external-repository-token");
+                extraArgs.push(`--codescanning-config=${codeScanningConfigFile}`);
                 if (externalRepositoryToken) {
                     extraArgs.push("--external-repository-token-stdin");
                 }
             }
+            if (await util.codeQlVersionAbove(this, exports.CODEQL_VERSION_INIT_WITH_QLCONFIG)) {
+                extraArgs.push(`--qlconfig=${qlconfigFile}`);
+            }
             await runTool(cmd, [
                 "database",
                 "init",
@@ -718,7 +725,7 @@ async function generateCodeScanningConfig(codeql, config, featureEnablement, log
     if (!(await util.useCodeScanningConfigInCli(codeql, featureEnablement))) {
         return;
     }
-    const configLocation = path.resolve(config.tempDir, "user-config.yaml");
+    const codeScanningConfigFile = path.resolve(config.tempDir, "user-config.yaml");
     // make a copy so we can modify it
     const augmentedConfig = cloneObject(config.originalUserInput);
     // Inject the queries from the input
@@ -772,12 +779,12 @@ async function generateCodeScanningConfig(codeql, config, featureEnablement, log
             augmentedConfig.packs["javascript"].push(packString);
         }
     }
-    logger.info(`Writing augmented user configuration file to ${configLocation}`);
+    logger.info(`Writing augmented user configuration file to ${codeScanningConfigFile}`);
     logger.startGroup("Augmented user configuration file contents");
     logger.info(yaml.dump(augmentedConfig));
     logger.endGroup();
-    fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
+    fs.writeFileSync(codeScanningConfigFile, yaml.dump(augmentedConfig));
-    return configLocation;
+    return codeScanningConfigFile;
 }
 function cloneObject(obj) {
     return JSON.parse(JSON.stringify(obj));

lib/codeql.test.js

@@ -452,11 +452,11 @@ for (const isBundleVersionInUrl of [true, false]) {
                 packsInputCombines: false,
             },
         };
-        await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
+        await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
         const args = runnerConstructorStub.firstCall.args[1];
         // should NOT have used an config file
         const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
-        t.falsy(configArg, "Should have injected a codescanning config");
+        t.falsy(configArg, "Should NOT have injected a codescanning config");
     });
 });
 // Test macro for ensuring different variants of injected augmented configurations
@@ -474,7 +474,7 @@ const injectedConfigMacro = ava_1.default.macro({
                 tempDir,
                 augmentationProperties,
             };
-            await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), (0, logging_1.getRunnerLogger)(true));
+            await codeqlObject.databaseInitCluster(thisStubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), undefined, (0, logging_1.getRunnerLogger)(true));
             const args = runnerConstructorStub.firstCall.args[1];
             // should have used an config file
             const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
@@ -665,24 +665,47 @@ const injectedConfigMacro = ava_1.default.macro({
         queries: [],
     },
 }, {});
-(0, ava_1.default)("does not use injected config", async (t) => {
+(0, ava_1.default)("does not pass a code scanning config or qlconfig file to the CLI when CLI config passing is disabled", async (t) => {
-    const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
+    const runnerConstructorStub = stubToolRunnerConstructor();
-    process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
+    const codeqlObject = await codeql.getCodeQLForTesting();
-    try {
+    // stubbed version doesn't matter. It just needs to be valid semver.
-        const runnerConstructorStub = stubToolRunnerConstructor();
+    sinon.stub(codeqlObject, "getVersion").resolves("0.0.0");
-        const codeqlObject = await codeql.getCodeQLForTesting();
+    await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
-        sinon
+    const args = runnerConstructorStub.firstCall.args[1];
-            .stub(codeqlObject, "getVersion")
+    // should not have used a config file
-            .resolves(feature_flags_1.featureConfig[feature_flags_1.Feature.CliConfigFileEnabled].minimumVersion);
+    const hasConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
-        await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([]), (0, logging_1.getRunnerLogger)(true));
+    t.false(hasConfigArg, "Should NOT have injected a codescanning config");
-        const args = runnerConstructorStub.firstCall.args[1];
+    // should not have passed a qlconfig file
-        // should have used an config file
+    const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
-        const configArg = args.find((arg) => arg.startsWith("--codescanning-config="));
+    t.false(hasQlconfigArg, "Should NOT have passed a qlconfig file");
-        t.falsy(configArg, "Should NOT have injected a codescanning config");
+});
-    }
+(0, ava_1.default)("passes a code scanning config AND qlconfig to the CLI when CLI config passing is enabled", async (t) => {
-    finally {
+    const runnerConstructorStub = stubToolRunnerConstructor();
-        process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
+    const codeqlObject = await codeql.getCodeQLForTesting();
-    }
+    sinon
+        .stub(codeqlObject, "getVersion")
+        .resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
+    await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
+    const args = runnerConstructorStub.firstCall.args[1];
+    // should have used a config file
+    const hasCodeScanningConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
+    t.true(hasCodeScanningConfigArg, "Should have injected a qlconfig");
+    // should have passed a qlconfig file
+    const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
+    t.truthy(hasQlconfigArg, "Should have injected a codescanning config");
+});
+(0, ava_1.default)("passes a code scanning config BUT NOT a qlconfig to the CLI when CLI config passing is enabled", async (t) => {
+    const runnerConstructorStub = stubToolRunnerConstructor();
+    const codeqlObject = await codeql.getCodeQLForTesting();
+    sinon.stub(codeqlObject, "getVersion").resolves("2.12.2");
+    await codeqlObject.databaseInitCluster(stubConfig, "", undefined, (0, testing_utils_1.createFeatures)([feature_flags_1.Feature.CliConfigFileEnabled]), "/path/to/qlconfig.yml", (0, logging_1.getRunnerLogger)(true));
+    const args = runnerConstructorStub.firstCall.args[1];
+    // should have used a config file
+    const hasCodeScanningConfigArg = args.some((arg) => arg.startsWith("--codescanning-config="));
+    t.true(hasCodeScanningConfigArg, "Should NOT have injected a qlconfig");
+    // should have passed a qlconfig file
+    const hasQlconfigArg = args.some((arg) => arg.startsWith("--qlconfig="));
+    t.false(hasQlconfigArg, "Should have injected a codescanning config");
 });
 (0, ava_1.default)("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {
     const runnerConstructorStub = stubToolRunnerConstructor();

lib/config-utils.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
+exports.wrapEnvironment = exports.generateRegistries = exports.downloadPacks = exports.getConfig = exports.getPathToParsedConfigFile = exports.initConfig = exports.parsePacks = exports.validatePackSpecification = exports.prettyPrintPack = exports.parsePacksSpecification = exports.parsePacksFromConfig = exports.calculateAugmentation = exports.getDefaultConfig = exports.getRawLanguages = exports.getLanguages = exports.getLanguagesInRepo = exports.getUnknownLanguagesError = exports.getNoLanguagesError = exports.getConfigFileDirectoryGivenMessage = exports.getConfigFileFormatInvalidMessage = exports.getConfigFileRepoFormatInvalidMessage = exports.getConfigFileDoesNotExistErrorMessage = exports.getConfigFileOutsideWorkspaceErrorMessage = exports.getLocalPathDoesNotExist = exports.getLocalPathOutsideOfRepository = exports.getPacksStrInvalid = exports.getPacksInvalid = exports.getPacksInvalidSplit = exports.getPathsInvalid = exports.getPathsIgnoreInvalid = exports.getQueryUsesInvalid = exports.getQueriesMissingUses = exports.getQueriesInvalid = exports.getDisableDefaultQueriesInvalid = exports.getNameInvalid = exports.validateAndSanitisePath = exports.defaultAugmentationProperties = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const perf_hooks_1 = require("perf_hooks");
@@ -149,7 +149,7 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, packs, suite
     }
     if (suiteName === "security-experimental" &&
         !(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE))) {
-        throw new Error(`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than 
+        throw new Error(`The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
       ${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
       ${codeql_1.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`);
     }
@@ -725,7 +725,7 @@ function parseQueriesFromInput(rawQueriesInput, queriesInputCombines) {
     }
     const trimmedInput = queriesInputCombines
         ? rawQueriesInput.trim().slice(1).trim()
-        : rawQueriesInput?.trim();
+        : rawQueriesInput?.trim() ?? "";
     if (queriesInputCombines && trimmedInput.length === 0) {
         throw new Error(getConfigFilePropertyError(undefined, "queries", "A '+' was used in the 'queries' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."));
     }
@@ -959,8 +959,7 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
                     "Please make sure that the default queries are enabled, or you are specifying queries to run.");
             }
         }
-        const registries = parseRegistries(registriesInput);
+        await downloadPacks(codeQL, config.languages, config.packs, apiDetails, registriesInput, config.tempDir, logger);
-        await downloadPacks(codeQL, config.languages, config.packs, registries, apiDetails, config.tempDir, logger);
     }
     // Save the config so we can easily access it again in the future
     await saveConfig(config, logger);
@@ -1056,21 +1055,9 @@ async function getConfig(tempDir, logger) {
     return JSON.parse(configString);
 }
 exports.getConfig = getConfig;
-async function downloadPacks(codeQL, languages, packs, registries, apiDetails, tmpDir, logger) {
+async function downloadPacks(codeQL, languages, packs, apiDetails, registriesInput, tempDir, logger) {
-    let qlconfigFile;
+    // This code path is only used when config parsing occurs in the Action.
-    let registriesAuthTokens;
+    const { registriesAuthTokens, qlconfigFile } = await generateRegistries(registriesInput, codeQL, tempDir, logger);
-    if (registries) {
-        if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
-            throw new Error(`'registries' input is not supported on CodeQL versions less than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`);
-        }
-        // generate a qlconfig.yml file to hold the registry configs.
-        const qlconfig = createRegistriesBlock(registries);
-        qlconfigFile = path.join(tmpDir, "qlconfig.yml");
-        fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
-        registriesAuthTokens = registries
-            .map((registry) => `${registry.url}=${registry.token}`)
-            .join(",");
-    }
     await wrapEnvironment({
         GITHUB_TOKEN: apiDetails.auth,
         CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
@@ -1098,6 +1085,48 @@ async function downloadPacks(codeQL, languages, packs, registries, apiDetails, t
     });
 }
 exports.downloadPacks = downloadPacks;
+/**
+ * Generate a `qlconfig.yml` file from the `registries` input.
+ * This file is used by the CodeQL CLI to list the registries to use for each
+ * pack.
+ *
+ * @param registriesInput The value of the `registries` input.
+ * @param codeQL a codeQL object, used only for checking the version of CodeQL.
+ * @param tempDir a temporary directory to store the generated qlconfig.yml file.
+ * @param logger a logger object.
+ * @returns The path to the generated `qlconfig.yml` file and the auth tokens to
+ *        use for each registry.
+ */
+async function generateRegistries(registriesInput, codeQL, tempDir, logger) {
+    const registries = parseRegistries(registriesInput);
+    let registriesAuthTokens;
+    let qlconfigFile;
+    if (registries) {
+        if (!(await (0, util_1.codeQlVersionAbove)(codeQL, codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD))) {
+            throw new Error(`The 'registries' input is not supported on CodeQL CLI versions earlier than ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD}. Please upgrade to CodeQL CLI version ${codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD} or later.`);
+        }
+        // generate a qlconfig.yml file to hold the registry configs.
+        const qlconfig = createRegistriesBlock(registries);
+        qlconfigFile = path.join(tempDir, "qlconfig.yml");
+        const qlconfigContents = yaml.dump(qlconfig);
+        fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
+        logger.debug("Generated qlconfig.yml:");
+        logger.debug(qlconfigContents);
+        registriesAuthTokens = registries
+            .map((registry) => `${registry.url}=${registry.token}`)
+            .join(",");
+    }
+    if (typeof process.env.CODEQL_REGISTRIES_AUTH === "string") {
+        logger.debug("Using CODEQL_REGISTRIES_AUTH environment variable to authenticate with registries.");
+    }
+    return {
+        registriesAuthTokens: 
+        // if the user has explicitly set the CODEQL_REGISTRIES_AUTH env var then use that
+        process.env.CODEQL_REGISTRIES_AUTH ?? registriesAuthTokens,
+        qlconfigFile,
+    };
+}
+exports.generateRegistries = generateRegistries;
 function createRegistriesBlock(registries) {
     if (!Array.isArray(registries) ||
         registries.some((r) => !r.url || !r.packages)) {
@@ -1147,4 +1176,5 @@ async function wrapEnvironment(env, operation) {
         }
     }
 }
+exports.wrapEnvironment = wrapEnvironment;
 //# sourceMappingURL=config-utils.js.map

lib/config-utils.test.js

@@ -1114,8 +1114,8 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
             java: ["a", "b"],
             go: ["c", "d"],
             python: ["e", "f"],
-        }, undefined, // registries
+        }, sampleApiDetails, undefined, // registriesAuthTokens
-        sampleApiDetails, tmpDir, logger);
+        tmpDir, logger);
         // Expecting packs to be downloaded once for java and once for python
         t.deepEqual(packDownloadStub.callCount, 2);
         // no config file was created, so pass `undefined` as the config file path
@@ -1128,9 +1128,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
     // associated env vars
     return await util.withTmpDir(async (tmpDir) => {
         process.env.GITHUB_TOKEN = "not-a-token";
-        process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
+        process.env.CODEQL_REGISTRIES_AUTH = undefined;
         const logger = (0, logging_1.getRunnerLogger)(true);
-        const registries = [
+        const registriesInput = yaml.dump([
             {
                 // no slash
                 url: "http://ghcr.io",
@@ -1143,8 +1143,9 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
                 packages: "semmle/*",
                 token: "still-not-a-token",
             },
-        ];
+        ]);
         // append a slash to the first url
+        const registries = yaml.load(registriesInput);
         const expectedRegistries = registries.map((r, i) => ({
             packages: r.packages,
             url: i === 0 ? `${r.url}/` : r.url,
@@ -1173,7 +1174,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
             java: ["a", "b"],
             go: ["c", "d"],
             python: ["e", "f"],
-        }, registries, sampleApiDetails, tmpDir, logger);
+        }, sampleApiDetails, registriesInput, tmpDir, logger);
         // Same packs are downloaded as in previous test
         t.deepEqual(packDownloadStub.callCount, 2);
         t.deepEqual(packDownloadStub.firstCall.args, [
@@ -1186,7 +1187,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
         ]);
         // Verify that the env vars were unset.
         t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
-        t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
+        t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, undefined);
     });
 });
 (0, ava_1.default)("downloadPacks-with-registries fails on 2.10.3", async (t) => {
@@ -1196,7 +1197,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
         process.env.GITHUB_TOKEN = "not-a-token";
         process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
         const logger = (0, logging_1.getRunnerLogger)(true);
-        const registries = [
+        const registriesInput = yaml.dump([
             {
                 url: "http://ghcr.io",
                 packages: ["codeql/*", "dsp-testing/*"],
@@ -1207,12 +1208,12 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
                 packages: "semmle/*",
                 token: "still-not-a-token",
             },
-        ];
+        ]);
         const codeQL = (0, codeql_1.setCodeQL)({
             getVersion: () => Promise.resolve("2.10.3"),
         });
         await t.throwsAsync(async () => {
-            return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
+            return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
         }, { instanceOf: Error }, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
     });
 });
@@ -1223,7 +1224,7 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
         process.env.GITHUB_TOKEN = "not-a-token";
         process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
         const logger = (0, logging_1.getRunnerLogger)(true);
-        const registries = [
+        const registriesInput = yaml.dump([
             {
                 // missing url property
                 packages: ["codeql/*", "dsp-testing/*"],
@@ -1234,15 +1235,68 @@ const calculateAugmentationErrorMacro = ava_1.default.macro({
                 packages: "semmle/*",
                 token: "still-not-a-token",
             },
-        ];
+        ]);
         const codeQL = (0, codeql_1.setCodeQL)({
             getVersion: () => Promise.resolve("2.10.4"),
         });
         await t.throwsAsync(async () => {
-            return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, registries, sampleApiDetails, tmpDir, logger);
+            return await configUtils.downloadPacks(codeQL, [languages_1.Language.javascript, languages_1.Language.java, languages_1.Language.python], {}, sampleApiDetails, registriesInput, tmpDir, logger);
         }, { instanceOf: Error }, "Invalid 'registries' input. Must be an array of objects with 'url' and 'packages' properties.");
     });
 });
+// the happy path for generateRegistries is already tested in downloadPacks.
+// these following tests are for the error cases and when nothing is generated.
+(0, ava_1.default)("no generateRegistries when CLI is too old", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const registriesInput = yaml.dump([
+            {
+                // no slash
+                url: "http://ghcr.io",
+                packages: ["codeql/*", "dsp-testing/*"],
+                token: "not-a-token",
+            },
+        ]);
+        const codeQL = (0, codeql_1.setCodeQL)({
+            // Accepted CLI versions are 2.10.4 or higher
+            getVersion: () => Promise.resolve("2.10.3"),
+        });
+        const logger = (0, logging_1.getRunnerLogger)(true);
+        await t.throwsAsync(async () => await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger), undefined, "'registries' input is not supported on CodeQL versions less than 2.10.4.");
+    });
+});
+(0, ava_1.default)("no generateRegistries when registries is undefined", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        const registriesInput = undefined;
+        const codeQL = (0, codeql_1.setCodeQL)({
+            // Accepted CLI versions are 2.10.4 or higher
+            getVersion: () => Promise.resolve(codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD),
+        });
+        const logger = (0, logging_1.getRunnerLogger)(true);
+        const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger);
+        t.is(registriesAuthTokens, undefined);
+        t.is(qlconfigFile, undefined);
+    });
+});
+(0, ava_1.default)("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        process.env.CODEQL_REGISTRIES_AUTH = "original";
+        const registriesInput = yaml.dump([
+            {
+                url: "http://ghcr.io",
+                packages: ["codeql/*", "dsp-testing/*"],
+                token: "not-a-token",
+            },
+        ]);
+        const codeQL = (0, codeql_1.setCodeQL)({
+            // Accepted CLI versions are 2.10.4 or higher
+            getVersion: () => Promise.resolve(codeql_1.CODEQL_VERSION_GHES_PACK_DOWNLOAD),
+        });
+        const logger = (0, logging_1.getRunnerLogger)(true);
+        const { registriesAuthTokens, qlconfigFile } = await configUtils.generateRegistries(registriesInput, codeQL, tmpDir, logger);
+        t.is(registriesAuthTokens, "original");
+        t.is(qlconfigFile, path.join(tmpDir, "qlconfig.yml"));
+    });
+});
 // getLanguages
 const mockRepositoryNwo = (0, repository_1.parseRepositoryNwo)("owner/repo");
 // eslint-disable-next-line github/array-foreach

lib/init-action.js

@@ -113,6 +113,7 @@ async function run() {
     const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
     (0, util_1.checkGitHubVersionInRange)(gitHubVersion, logger);
     const repositoryNwo = (0, repository_1.parseRepositoryNwo)((0, util_1.getRequiredEnvParam)("GITHUB_REPOSITORY"));
+    const registriesInput = (0, actions_util_1.getOptionalInput)("registries");
     const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, (0, actions_util_1.getTemporaryDirectory)(), logger);
     try {
         const workflowErrors = await (0, workflow_1.validateWorkflow)();
@@ -129,7 +130,7 @@ async function run() {
         toolsVersion = initCodeQLResult.toolsVersion;
         toolsSource = initCodeQLResult.toolsSource;
         await (0, util_1.enrichEnvironment)(codeql);
-        config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("registries"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(), 
+        config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), registriesInput, (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), getTrapCachingEnabled(), 
         // Debug mode is enabled if:
         // - The `init` Action is passed `debug: true`.
         // - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
@@ -173,7 +174,7 @@ async function run() {
             core.exportVariable("CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN", "true");
         }
         const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
-        const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", features, logger);
+        const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", registriesInput, features, apiDetails, logger);
         if (tracerConfig !== undefined) {
             for (const [key, value] of Object.entries(tracerConfig.env)) {
                 core.exportVariable(key, value);

lib/init.js

@@ -57,12 +57,26 @@ async function initConfig(languagesInput, queriesInput, packsInput, registriesIn
     return config;
 }
 exports.initConfig = initConfig;
-async function runInit(codeql, config, sourceRoot, processName, featureEnablement, logger) {
+async function runInit(codeql, config, sourceRoot, processName, registriesInput, featureEnablement, apiDetails, logger) {
     fs.mkdirSync(config.dbLocation, { recursive: true });
     try {
         if (await (0, util_1.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
+            // When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
+            // So, create it now.
+            // If we are parsing the config file in the Action, then the qlconfig file was already created
+            // before the `pack download` command was invoked. It is not required for the init command.
+            let registriesAuthTokens;
+            let qlconfigFile;
+            if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
+                ({ registriesAuthTokens, qlconfigFile } =
+                    await configUtils.generateRegistries(registriesInput, codeql, config.tempDir, logger));
+            }
+            await configUtils.wrapEnvironment({
+                GITHUB_TOKEN: apiDetails.auth,
+                CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
+            }, 
             // Init a database cluster
-            await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, logger);
+            async () => await codeql.databaseInitCluster(config, sourceRoot, processName, featureEnablement, qlconfigFile, logger));
         }
         else {
             for (const language of config.languages) {

node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "codeql",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "codeql",
-      "version": "2.2.5",
+      "version": "2.2.6",
       "license": "MIT",
       "dependencies": {
         "@actions/artifact": "^1.1.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "2.2.5",
+  "version": "2.2.6",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

pr-checks/checks/init-with-registries.yml

@@ -4,7 +4,12 @@
 # basic mechanics of multi-registry auth is working.
 name: "Packaging: Download using registries"
 description: "Checks that specifying a registries block and associated auth works as expected"
-versions: ["nightly-latest"] # This feature is not compatible with old CLIs
+versions: [
+    # This feature is not compatible with older CLIs
+    "cached",
+    "latest",
+    "nightly-latest",
+]
 
 steps:
   - name: Init with registries
@@ -40,3 +45,33 @@ steps:
           echo "::error $CODEQL_PACK1 pack was not installed."
           exit 1
       fi
+
+  - name: Verify qlconfig.yml file was created
+    shell: bash
+    run: |
+      QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
+      echo "Expected qlconfig.yml file to be created at $QLCONFIG_PATH"
+      if [[ -f $QLCONFIG_PATH ]]
+      then
+          echo "qlconfig.yml file was created."
+      else
+          echo "::error qlconfig.yml file was not created."
+          exit 1
+      fi
+
+  - name: Verify contents of qlconfig.yml
+    # yq is not available on windows
+    if: runner.os != 'Windows'
+    shell: bash
+    run: |
+      QLCONFIG_PATH=$RUNNER_TEMP/qlconfig.yml
+      cat $QLCONFIG_PATH | yq -e '.registries[] | select(.url == "https://ghcr.io/v2/") | select(.packages == "*/*")'
+      if [[ $? -eq 0 ]]
+      then
+          echo "Registry was added to qlconfig.yml file."
+      else
+          echo "::error Registry was not added to qlconfig.yml file."
+          echo "Contents of qlconfig.yml file:"
+          cat $QLCONFIG_PATH
+          exit 1
+      fi

src/codeql.test.ts

@@ -671,6 +671,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
       "",
       undefined,
       createFeatures([]),
+      "/path/to/qlconfig.yml",
       getRunnerLogger(true)
     );
 
@@ -679,7 +680,7 @@ test("databaseInitCluster() without injected codescanning config", async (t) =>
     const configArg = args.find((arg: string) =>
       arg.startsWith("--codescanning-config=")
     );
-    t.falsy(configArg, "Should have injected a codescanning config");
+    t.falsy(configArg, "Should NOT have injected a codescanning config");
   });
 });
 
@@ -710,6 +711,7 @@ const injectedConfigMacro = test.macro({
         "",
         undefined,
         createFeatures([Feature.CliConfigFileEnabled]),
+        undefined,
         getRunnerLogger(true)
       );
 
@@ -1000,34 +1002,90 @@ test(
   {}
 );
 
-test("does not use injected config", async (t: ExecutionContext<unknown>) => {
+test("does not pass a code scanning config or qlconfig file to the CLI when CLI config passing is disabled", async (t: ExecutionContext<unknown>) => {
-  const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
+  const runnerConstructorStub = stubToolRunnerConstructor();
-  process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";
+  const codeqlObject = await codeql.getCodeQLForTesting();
+  // stubbed version doesn't matter. It just needs to be valid semver.
+  sinon.stub(codeqlObject, "getVersion").resolves("0.0.0");
 
-  try {
+  await codeqlObject.databaseInitCluster(
-    const runnerConstructorStub = stubToolRunnerConstructor();
+    stubConfig,
-    const codeqlObject = await codeql.getCodeQLForTesting();
+    "",
-    sinon
+    undefined,
-      .stub(codeqlObject, "getVersion")
+    createFeatures([]),
-      .resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
+    "/path/to/qlconfig.yml",
+    getRunnerLogger(true)
+  );
 
-    await codeqlObject.databaseInitCluster(
+  const args = runnerConstructorStub.firstCall.args[1];
-      stubConfig,
+  // should not have used a config file
-      "",
+  const hasConfigArg = args.some((arg: string) =>
-      undefined,
+    arg.startsWith("--codescanning-config=")
-      createFeatures([]),
+  );
-      getRunnerLogger(true)
+  t.false(hasConfigArg, "Should NOT have injected a codescanning config");
-    );
 
-    const args = runnerConstructorStub.firstCall.args[1];
+  // should not have passed a qlconfig file
-    // should have used an config file
+  const hasQlconfigArg = args.some((arg: string) =>
-    const configArg = args.find((arg: string) =>
+    arg.startsWith("--qlconfig=")
-      arg.startsWith("--codescanning-config=")
+  );
-    );
+  t.false(hasQlconfigArg, "Should NOT have passed a qlconfig file");
-    t.falsy(configArg, "Should NOT have injected a codescanning config");
+});
-  } finally {
+
-    process.env["CODEQL_PASS_CONFIG_TO_CLI"] = origCODEQL_PASS_CONFIG_TO_CLI;
+test("passes a code scanning config AND qlconfig to the CLI when CLI config passing is enabled", async (t: ExecutionContext<unknown>) => {
-  }
+  const runnerConstructorStub = stubToolRunnerConstructor();
+  const codeqlObject = await codeql.getCodeQLForTesting();
+  sinon
+    .stub(codeqlObject, "getVersion")
+    .resolves(codeql.CODEQL_VERSION_INIT_WITH_QLCONFIG);
+
+  await codeqlObject.databaseInitCluster(
+    stubConfig,
+    "",
+    undefined,
+    createFeatures([Feature.CliConfigFileEnabled]),
+    "/path/to/qlconfig.yml",
+    getRunnerLogger(true)
+  );
+
+  const args = runnerConstructorStub.firstCall.args[1];
+  // should have used a config file
+  const hasCodeScanningConfigArg = args.some((arg: string) =>
+    arg.startsWith("--codescanning-config=")
+  );
+  t.true(hasCodeScanningConfigArg, "Should have injected a qlconfig");
+
+  // should have passed a qlconfig file
+  const hasQlconfigArg = args.some((arg: string) =>
+    arg.startsWith("--qlconfig=")
+  );
+  t.truthy(hasQlconfigArg, "Should have injected a codescanning config");
+});
+test("passes a code scanning config BUT NOT a qlconfig to the CLI when CLI config passing is enabled", async (t: ExecutionContext<unknown>) => {
+  const runnerConstructorStub = stubToolRunnerConstructor();
+  const codeqlObject = await codeql.getCodeQLForTesting();
+  sinon.stub(codeqlObject, "getVersion").resolves("2.12.2");
+
+  await codeqlObject.databaseInitCluster(
+    stubConfig,
+    "",
+    undefined,
+    createFeatures([Feature.CliConfigFileEnabled]),
+    "/path/to/qlconfig.yml",
+    getRunnerLogger(true)
+  );
+
+  const args = runnerConstructorStub.firstCall.args[1] as any[];
+  // should have used a config file
+  const hasCodeScanningConfigArg = args.some((arg: string) =>
+    arg.startsWith("--codescanning-config=")
+  );
+  t.true(hasCodeScanningConfigArg, "Should NOT have injected a qlconfig");
+
+  // should have passed a qlconfig file
+  const hasQlconfigArg = args.some((arg: string) =>
+    arg.startsWith("--qlconfig=")
+  );
+  t.false(hasQlconfigArg, "Should have injected a codescanning config");
 });
 
 test("databaseInterpretResults() sets --sarif-add-baseline-file-info for 2.11.3", async (t) => {

src/codeql.ts

@@ -91,6 +91,7 @@ export interface CodeQL {
     sourceRoot: string,
     processName: string | undefined,
     featureEnablement: FeatureEnablement,
+    qlconfigFile: string | undefined,
     logger: Logger
   ): Promise<void>;
   /**
@@ -289,6 +290,11 @@ export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
  */
 export const CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = "2.12.1";
 
+/**
+ * Versions 2.12.4+ of the CodeQL CLI support the `--qlconfig` flag in calls to `database init`.
+ */
+export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
+
 /**
  * Set up CodeQL CLI access.
  *
@@ -565,6 +571,7 @@ export async function getCodeQLForCmd(
       sourceRoot: string,
       processName: string | undefined,
       featureEnablement: FeatureEnablement,
+      qlconfigFile: string | undefined,
       logger: Logger
     ) {
       const extraArgs = config.languages.map(
@@ -594,8 +601,8 @@ export async function getCodeQLForCmd(
         }
       }
 
-      // A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
+      // A code scanning config file is only generated if the CliConfigFileEnabled feature flag is enabled.
-      const configLocation = await generateCodeScanningConfig(
+      const codeScanningConfigFile = await generateCodeScanningConfig(
         codeql,
         config,
         featureEnablement,
@@ -603,14 +610,19 @@ export async function getCodeQLForCmd(
       );
       // Only pass external repository token if a config file is going to be parsed by the CLI.
       let externalRepositoryToken: string | undefined;
-      if (configLocation) {
+      if (codeScanningConfigFile) {
-        extraArgs.push(`--codescanning-config=${configLocation}`);
         externalRepositoryToken = getOptionalInput("external-repository-token");
+        extraArgs.push(`--codescanning-config=${codeScanningConfigFile}`);
         if (externalRepositoryToken) {
           extraArgs.push("--external-repository-token-stdin");
         }
       }
 
+      if (
+        await util.codeQlVersionAbove(this, CODEQL_VERSION_INIT_WITH_QLCONFIG)
+      ) {
+        extraArgs.push(`--qlconfig=${qlconfigFile}`);
+      }
       await runTool(
         cmd,
         [
@@ -1111,7 +1123,10 @@ async function generateCodeScanningConfig(
   if (!(await util.useCodeScanningConfigInCli(codeql, featureEnablement))) {
     return;
   }
-  const configLocation = path.resolve(config.tempDir, "user-config.yaml");
+  const codeScanningConfigFile = path.resolve(
+    config.tempDir,
+    "user-config.yaml"
+  );
   // make a copy so we can modify it
   const augmentedConfig = cloneObject(config.originalUserInput);
 
@@ -1168,13 +1183,15 @@ async function generateCodeScanningConfig(
       augmentedConfig.packs["javascript"].push(packString);
     }
   }
-  logger.info(`Writing augmented user configuration file to ${configLocation}`);
+  logger.info(
+    `Writing augmented user configuration file to ${codeScanningConfigFile}`
+  );
   logger.startGroup("Augmented user configuration file contents");
   logger.info(yaml.dump(augmentedConfig));
   logger.endGroup();
 
-  fs.writeFileSync(configLocation, yaml.dump(augmentedConfig));
+  fs.writeFileSync(codeScanningConfigFile, yaml.dump(augmentedConfig));
-  return configLocation;
+  return codeScanningConfigFile;
 }
 
 function cloneObject<T>(obj: T): T {

src/config-utils.test.ts

@@ -7,9 +7,13 @@ import * as yaml from "js-yaml";
 import * as sinon from "sinon";
 
 import * as api from "./api-client";
-import { getCachedCodeQL, PackDownloadOutput, setCodeQL } from "./codeql";
+import {
+  CODEQL_VERSION_GHES_PACK_DOWNLOAD,
+  getCachedCodeQL,
+  PackDownloadOutput,
+  setCodeQL,
+} from "./codeql";
 import * as configUtils from "./config-utils";
-import { RegistryConfigWithCredentials } from "./config-utils";
 import { Feature } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger, Logger } from "./logging";
@@ -2277,8 +2281,8 @@ test("downloadPacks-no-registries", async (t) => {
         go: ["c", "d"],
         python: ["e", "f"],
       },
-      undefined, // registries
       sampleApiDetails,
+      undefined, // registriesAuthTokens
       tmpDir,
       logger
     );
@@ -2296,10 +2300,10 @@ test("downloadPacks-with-registries", async (t) => {
   // associated env vars
   return await util.withTmpDir(async (tmpDir) => {
     process.env.GITHUB_TOKEN = "not-a-token";
-    process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
+    process.env.CODEQL_REGISTRIES_AUTH = undefined;
     const logger = getRunnerLogger(true);
 
-    const registries = [
+    const registriesInput = yaml.dump([
       {
         // no slash
         url: "http://ghcr.io",
@@ -2312,9 +2316,12 @@ test("downloadPacks-with-registries", async (t) => {
         packages: "semmle/*",
         token: "still-not-a-token",
       },
-    ];
+    ]);
 
     // append a slash to the first url
+    const registries = yaml.load(
+      registriesInput
+    ) as configUtils.RegistryConfigWithCredentials[];
     const expectedRegistries = registries.map((r, i) => ({
       packages: r.packages,
       url: i === 0 ? `${r.url}/` : r.url,
@@ -2356,8 +2363,8 @@ test("downloadPacks-with-registries", async (t) => {
         go: ["c", "d"],
         python: ["e", "f"],
       },
-      registries,
       sampleApiDetails,
+      registriesInput,
       tmpDir,
       logger
     );
@@ -2375,7 +2382,7 @@ test("downloadPacks-with-registries", async (t) => {
 
     // Verify that the env vars were unset.
     t.deepEqual(process.env.GITHUB_TOKEN, "not-a-token");
-    t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, "not-a-registries-auth");
+    t.deepEqual(process.env.CODEQL_REGISTRIES_AUTH, undefined);
   });
 });
 
@@ -2387,7 +2394,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
     process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
     const logger = getRunnerLogger(true);
 
-    const registries = [
+    const registriesInput = yaml.dump([
       {
         url: "http://ghcr.io",
         packages: ["codeql/*", "dsp-testing/*"],
@@ -2398,7 +2405,7 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
         packages: "semmle/*",
         token: "still-not-a-token",
       },
-    ];
+    ]);
 
     const codeQL = setCodeQL({
       getVersion: () => Promise.resolve("2.10.3"),
@@ -2409,8 +2416,8 @@ test("downloadPacks-with-registries fails on 2.10.3", async (t) => {
           codeQL,
           [Language.javascript, Language.java, Language.python],
           {},
-          registries,
           sampleApiDetails,
+          registriesInput,
           tmpDir,
           logger
         );
@@ -2429,7 +2436,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
     process.env.CODEQL_REGISTRIES_AUTH = "not-a-registries-auth";
     const logger = getRunnerLogger(true);
 
-    const registries = [
+    const registriesInput = yaml.dump([
       {
         // missing url property
         packages: ["codeql/*", "dsp-testing/*"],
@@ -2440,7 +2447,7 @@ test("downloadPacks-with-registries fails with invalid registries block", async
         packages: "semmle/*",
         token: "still-not-a-token",
       },
-    ];
+    ]);
 
     const codeQL = setCodeQL({
       getVersion: () => Promise.resolve("2.10.4"),
@@ -2451,8 +2458,8 @@ test("downloadPacks-with-registries fails with invalid registries block", async
           codeQL,
           [Language.javascript, Language.java, Language.python],
           {},
-          registries as RegistryConfigWithCredentials[] | undefined,
           sampleApiDetails,
+          registriesInput,
           tmpDir,
           logger
         );
@@ -2463,6 +2470,85 @@ test("downloadPacks-with-registries fails with invalid registries block", async
   });
 });
 
+// the happy path for generateRegistries is already tested in downloadPacks.
+// these following tests are for the error cases and when nothing is generated.
+test("no generateRegistries when CLI is too old", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const registriesInput = yaml.dump([
+      {
+        // no slash
+        url: "http://ghcr.io",
+        packages: ["codeql/*", "dsp-testing/*"],
+        token: "not-a-token",
+      },
+    ]);
+    const codeQL = setCodeQL({
+      // Accepted CLI versions are 2.10.4 or higher
+      getVersion: () => Promise.resolve("2.10.3"),
+    });
+    const logger = getRunnerLogger(true);
+    await t.throwsAsync(
+      async () =>
+        await configUtils.generateRegistries(
+          registriesInput,
+          codeQL,
+          tmpDir,
+          logger
+        ),
+      undefined,
+      "'registries' input is not supported on CodeQL versions less than 2.10.4."
+    );
+  });
+});
+test("no generateRegistries when registries is undefined", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    const registriesInput = undefined;
+    const codeQL = setCodeQL({
+      // Accepted CLI versions are 2.10.4 or higher
+      getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
+    });
+    const logger = getRunnerLogger(true);
+    const { registriesAuthTokens, qlconfigFile } =
+      await configUtils.generateRegistries(
+        registriesInput,
+        codeQL,
+        tmpDir,
+        logger
+      );
+
+    t.is(registriesAuthTokens, undefined);
+    t.is(qlconfigFile, undefined);
+  });
+});
+
+test("generateRegistries prefers original CODEQL_REGISTRIES_AUTH", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    process.env.CODEQL_REGISTRIES_AUTH = "original";
+    const registriesInput = yaml.dump([
+      {
+        url: "http://ghcr.io",
+        packages: ["codeql/*", "dsp-testing/*"],
+        token: "not-a-token",
+      },
+    ]);
+    const codeQL = setCodeQL({
+      // Accepted CLI versions are 2.10.4 or higher
+      getVersion: () => Promise.resolve(CODEQL_VERSION_GHES_PACK_DOWNLOAD),
+    });
+    const logger = getRunnerLogger(true);
+    const { registriesAuthTokens, qlconfigFile } =
+      await configUtils.generateRegistries(
+        registriesInput,
+        codeQL,
+        tmpDir,
+        logger
+      );
+
+    t.is(registriesAuthTokens, "original");
+    t.is(qlconfigFile, path.join(tmpDir, "qlconfig.yml"));
+  });
+});
+
 // getLanguages
 
 const mockRepositoryNwo = parseRepositoryNwo("owner/repo");

src/config-utils.ts

@@ -414,7 +414,7 @@ async function addBuiltinSuiteQueries(
     ))
   ) {
     throw new Error(
-      `The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than 
+      `The 'security-experimental' suite is not supported on CodeQL CLI versions earlier than
       ${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE}. Please upgrade to CodeQL CLI version
       ${CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE} or later.`
     );
@@ -1403,7 +1403,7 @@ function parseQueriesFromInput(
 
   const trimmedInput = queriesInputCombines
     ? rawQueriesInput.trim().slice(1).trim()
-    : rawQueriesInput?.trim();
+    : rawQueriesInput?.trim() ?? "";
   if (queriesInputCombines && trimmedInput.length === 0) {
     throw new Error(
       getConfigFilePropertyError(
@@ -1769,13 +1769,12 @@ export async function initConfig(
       }
     }
 
-    const registries = parseRegistries(registriesInput);
     await downloadPacks(
       codeQL,
       config.languages,
       config.packs,
-      registries,
       apiDetails,
+      registriesInput,
       config.tempDir,
       logger
     );
@@ -1899,32 +1898,18 @@ export async function downloadPacks(
   codeQL: CodeQL,
   languages: Language[],
   packs: Packs,
-  registries: RegistryConfigWithCredentials[] | undefined,
   apiDetails: api.GitHubApiDetails,
-  tmpDir: string,
+  registriesInput: string | undefined,
+  tempDir: string,
   logger: Logger
 ) {
-  let qlconfigFile: string | undefined;
+  // This code path is only used when config parsing occurs in the Action.
-  let registriesAuthTokens: string | undefined;
+  const { registriesAuthTokens, qlconfigFile } = await generateRegistries(
-  if (registries) {
+    registriesInput,
-    if (
+    codeQL,
-      !(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
+    tempDir,
-    ) {
+    logger
-      throw new Error(
+  );
-        `'registries' input is not supported on CodeQL versions less than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}.`
-      );
-    }
-
-    // generate a qlconfig.yml file to hold the registry configs.
-    const qlconfig = createRegistriesBlock(registries);
-    qlconfigFile = path.join(tmpDir, "qlconfig.yml");
-    fs.writeFileSync(qlconfigFile, yaml.dump(qlconfig), "utf8");
-
-    registriesAuthTokens = registries
-      .map((registry) => `${registry.url}=${registry.token}`)
-      .join(",");
-  }
-
   await wrapEnvironment(
     {
       GITHUB_TOKEN: apiDetails.auth,
@@ -1963,6 +1948,63 @@ export async function downloadPacks(
   );
 }
 
+/**
+ * Generate a `qlconfig.yml` file from the `registries` input.
+ * This file is used by the CodeQL CLI to list the registries to use for each
+ * pack.
+ *
+ * @param registriesInput The value of the `registries` input.
+ * @param codeQL a codeQL object, used only for checking the version of CodeQL.
+ * @param tempDir a temporary directory to store the generated qlconfig.yml file.
+ * @param logger a logger object.
+ * @returns The path to the generated `qlconfig.yml` file and the auth tokens to
+ *        use for each registry.
+ */
+export async function generateRegistries(
+  registriesInput: string | undefined,
+  codeQL: CodeQL,
+  tempDir: string,
+  logger: Logger
+) {
+  const registries = parseRegistries(registriesInput);
+  let registriesAuthTokens: string | undefined;
+  let qlconfigFile: string | undefined;
+  if (registries) {
+    if (
+      !(await codeQlVersionAbove(codeQL, CODEQL_VERSION_GHES_PACK_DOWNLOAD))
+    ) {
+      throw new Error(
+        `The 'registries' input is not supported on CodeQL CLI versions earlier than ${CODEQL_VERSION_GHES_PACK_DOWNLOAD}. Please upgrade to CodeQL CLI version ${CODEQL_VERSION_GHES_PACK_DOWNLOAD} or later.`
+      );
+    }
+
+    // generate a qlconfig.yml file to hold the registry configs.
+    const qlconfig = createRegistriesBlock(registries);
+    qlconfigFile = path.join(tempDir, "qlconfig.yml");
+    const qlconfigContents = yaml.dump(qlconfig);
+    fs.writeFileSync(qlconfigFile, qlconfigContents, "utf8");
+
+    logger.debug("Generated qlconfig.yml:");
+    logger.debug(qlconfigContents);
+    registriesAuthTokens = registries
+      .map((registry) => `${registry.url}=${registry.token}`)
+      .join(",");
+  }
+
+  if (typeof process.env.CODEQL_REGISTRIES_AUTH === "string") {
+    logger.debug(
+      "Using CODEQL_REGISTRIES_AUTH environment variable to authenticate with registries."
+    );
+  }
+
+  return {
+    registriesAuthTokens:
+      // if the user has explicitly set the CODEQL_REGISTRIES_AUTH env var then use that
+      process.env.CODEQL_REGISTRIES_AUTH ?? registriesAuthTokens,
+    qlconfigFile,
+  };
+}
+
 function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
   registries: RegistryConfigNoCredentials[];
 } {
@@ -1999,7 +2041,7 @@ function createRegistriesBlock(registries: RegistryConfigWithCredentials[]): {
  * @param env
  * @param operation
  */
-async function wrapEnvironment(
+export async function wrapEnvironment(
   env: Record<string, string | undefined>,
   operation: Function
 ) {

src/init-action.ts

@@ -204,6 +204,8 @@ async function run() {
     getRequiredEnvParam("GITHUB_REPOSITORY")
   );
 
+  const registriesInput = getOptionalInput("registries");
+
   const features = new Features(
     gitHubVersion,
     repositoryNwo,
@@ -251,7 +253,7 @@ async function run() {
       getOptionalInput("languages"),
       getOptionalInput("queries"),
       getOptionalInput("packs"),
-      getOptionalInput("registries"),
+      registriesInput,
       getOptionalInput("config-file"),
       getOptionalInput("db-location"),
       getTrapCachingEnabled(),
@@ -335,7 +337,9 @@ async function run() {
       config,
       sourceRoot,
       "Runner.Worker.exe",
+      registriesInput,
       features,
+      apiDetails,
       logger
     );
     if (tracerConfig !== undefined) {

src/init.ts

@@ -102,20 +102,46 @@ export async function runInit(
   config: configUtils.Config,
   sourceRoot: string,
   processName: string | undefined,
+  registriesInput: string | undefined,
   featureEnablement: FeatureEnablement,
+  apiDetails: GitHubApiCombinedDetails,
   logger: Logger
 ): Promise<TracerConfig | undefined> {
   fs.mkdirSync(config.dbLocation, { recursive: true });
 
   try {
     if (await codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
-      // Init a database cluster
+      // When parsing the codeql config in the CLI, we have not yet created the qlconfig file.
-      await codeql.databaseInitCluster(
+      // So, create it now.
-        config,
+      // If we are parsing the config file in the Action, then the qlconfig file was already created
-        sourceRoot,
+      // before the `pack download` command was invoked. It is not required for the init command.
-        processName,
+      let registriesAuthTokens: string | undefined;
-        featureEnablement,
+      let qlconfigFile: string | undefined;
-        logger
+      if (await util.useCodeScanningConfigInCli(codeql, featureEnablement)) {
+        ({ registriesAuthTokens, qlconfigFile } =
+          await configUtils.generateRegistries(
+            registriesInput,
+            codeql,
+            config.tempDir,
+            logger
+          ));
+      }
+      await configUtils.wrapEnvironment(
+        {
+          GITHUB_TOKEN: apiDetails.auth,
+          CODEQL_REGISTRIES_AUTH: registriesAuthTokens,
+        },
+
+        // Init a database cluster
+        async () =>
+          await codeql.databaseInitCluster(
+            config,
+            sourceRoot,
+            processName,
+            featureEnablement,
+            qlconfigFile,
+            logger
+          )
       );
     } else {
       for (const language of config.languages) {