diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0effe3b0..4a9904eb 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -21,8 +21,8 @@ on: - cron: '15 16 * * 2' env: - NPM_CONFIG_REGISTRY: https://packagefeedproxy.microsoft.io/npm/ - PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/ + NPM_CONFIG_REGISTRY: https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/ + PIP_INDEX_URL: https://pkgs.dev.azure.com/devdiv/DevDiv/_packaging/debugpy/pypi/simple/ jobs: analyze: diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 47812b0c..be16b7ed 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -6,7 +6,7 @@ on: permissions: {} env: - PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/ + PIP_INDEX_URL: https://pkgs.dev.azure.com/devdiv/DevDiv/_packaging/debugpy/pypi/simple/ NODE_VERSION: 22.17.0 TEST_RESULTS_DIRECTORY: . # Force a path with spaces and unicode chars to test extension works in these scenarios diff --git a/.github/workflows/push-check.yml b/.github/workflows/push-check.yml index c958f625..dfa15494 100644 --- a/.github/workflows/push-check.yml +++ b/.github/workflows/push-check.yml @@ -11,7 +11,7 @@ on: permissions: {} env: - PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/ + PIP_INDEX_URL: https://pkgs.dev.azure.com/devdiv/DevDiv/_packaging/debugpy/pypi/simple/ NODE_VERSION: 22.17.0 TEST_RESULTS_DIRECTORY: . # Force a path with spaces and unicode chars to test extension works in these scenarios diff --git a/build/azure-devdiv-pipeline.pre-release.yml b/build/azure-devdiv-pipeline.pre-release.yml index 8ad15e48..df51f4af 100644 --- a/build/azure-devdiv-pipeline.pre-release.yml +++ b/build/azure-devdiv-pipeline.pre-release.yml @@ -34,9 +34,10 @@ variables: - name: VsixName value: python-debugger.vsix - name: NPM_CONFIG_REGISTRY - value: 'https://packagefeedproxy.microsoft.io/npm/' + value: 'https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/' - name: PIP_PROXY_INDEX_URL - value: 'https://packagefeedproxy.microsoft.io/pypi/simple/' + # PipAuthenticate supplies the internal index URL with credentials before buildSteps. + value: '$(PIP_INDEX_URL)' parameters: - name: publishExtension diff --git a/build/azure-devdiv-pipeline.stable.yml b/build/azure-devdiv-pipeline.stable.yml index feaccba7..c13fdfa9 100644 --- a/build/azure-devdiv-pipeline.stable.yml +++ b/build/azure-devdiv-pipeline.stable.yml @@ -29,9 +29,10 @@ variables: - name: VsixName value: python-debugger.vsix - name: NPM_CONFIG_REGISTRY - value: 'https://packagefeedproxy.microsoft.io/npm/' + value: 'https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/' - name: PIP_PROXY_INDEX_URL - value: 'https://packagefeedproxy.microsoft.io/pypi/simple/' + # PipAuthenticate supplies the internal index URL with credentials before buildSteps. + value: '$(PIP_INDEX_URL)' parameters: - name: publishExtension diff --git a/build/package-feeds.json b/build/package-feeds.json index c32a71a0..75592b5e 100644 --- a/build/package-feeds.json +++ b/build/package-feeds.json @@ -4,7 +4,7 @@ "pip": "https://pypi.org/simple/" }, "microsoft": { - "npm": "https://packagefeedproxy.microsoft.io/npm/", - "pip": "https://packagefeedproxy.microsoft.io/pypi/simple/" + "npm": "https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/", + "pip": "https://pkgs.dev.azure.com/devdiv/DevDiv/_packaging/debugpy/pypi/simple/" } } diff --git a/build/setup-npm-registry.js b/build/setup-npm-registry.js index 8f084651..f902c595 100644 --- a/build/setup-npm-registry.js +++ b/build/setup-npm-registry.js @@ -6,7 +6,11 @@ const { readFileSync, writeFileSync } = require('node:fs'); const path = require('node:path'); const { microsoft } = require('./package-feeds.json'); -function rewritePackageLock(source) { +function normalizeRegistry(registry) { + return registry.endsWith('/') ? registry : `${registry}/`; +} + +function rewritePackageLock(source, registry = microsoft.npm) { const lockfile = JSON.parse(source); if ( ![2, 3].includes(lockfile.lockfileVersion) || @@ -17,18 +21,19 @@ function rewritePackageLock(source) { throw new Error('Expected an npm lockfile with lockfileVersion 2 or 3 and a packages section.'); } + const normalizedRegistry = normalizeRegistry(registry); return source.replace(/("resolved"\s*:\s*)("(?:\\.|[^"\\])*")/g, (match, prefix, encodedUrl) => { const url = JSON.parse(encodedUrl); - const rewritten = url.replace(/^https:\/\/registry\.(?:npmjs\.org|yarnpkg\.com)\//, microsoft.npm); + const rewritten = url.replace(/^https:\/\/registry\.(?:npmjs\.org|yarnpkg\.com)\//, normalizedRegistry); return rewritten === url ? match : prefix + JSON.stringify(rewritten); }); } -function rewritePackageLocks(root = path.resolve(__dirname, '..')) { +function rewritePackageLocks(root = path.resolve(__dirname, '..'), registry = microsoft.npm) { const files = [path.join(root, 'package-lock.json')]; const updates = files.map((file) => { const source = readFileSync(file, 'utf8'); - return { file, source, rewritten: rewritePackageLock(source) }; + return { file, source, rewritten: rewritePackageLock(source, registry) }; }); for (const { file, source, rewritten } of updates) { if (rewritten !== source) { @@ -48,13 +53,15 @@ if (require.main === module) { if (install && !process.env.npm_execpath) { throw new Error('Run npm install using npm run ci:msft.'); } - rewritePackageLocks(); + const registry = normalizeRegistry( + process.env.npm_config_registry || process.env.NPM_CONFIG_REGISTRY || microsoft.npm, + ); + rewritePackageLocks(undefined, registry); if (install) { - const result = spawnSync( - process.execPath, - [process.env.npm_execpath, 'ci', `--registry=${microsoft.npm}`], - { cwd: path.resolve(__dirname, '..'), stdio: 'inherit' }, - ); + const result = spawnSync(process.execPath, [process.env.npm_execpath, 'ci', `--registry=${registry}`], { + cwd: path.resolve(__dirname, '..'), + stdio: 'inherit', + }); if (result.error || result.signal || result.status !== 0) { const reason = result.error?.message || result.signal || `exit code ${result.status}`; const error = new Error(`npm ci failed: ${reason}`); diff --git a/build/setup-npm-registry.test.js b/build/setup-npm-registry.test.js index 3d7eae5c..485cb169 100644 --- a/build/setup-npm-registry.test.js +++ b/build/setup-npm-registry.test.js @@ -11,7 +11,7 @@ const { rewritePackageLock, rewritePackageLocks } = require('./setup-npm-registr const { normalizePackageLock } = require('./normalize-package-lock'); const publicRegistry = 'https://registry.npmjs.org/'; -const proxyRegistry = 'https://packagefeedproxy.microsoft.io/npm/'; +const microsoftRegistry = 'https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/'; const source = `${JSON.stringify( { lockfileVersion: 3, @@ -34,20 +34,27 @@ const source = `${JSON.stringify( test('rewrites public resolved URLs without changing versions, integrity or formatting', () => { const expected = source - .replaceAll(publicRegistry, proxyRegistry) - .replace('https://registry.yarnpkg.com/', proxyRegistry); + .replaceAll(publicRegistry, microsoftRegistry) + .replace('https://registry.yarnpkg.com/', microsoftRegistry); assert.equal(rewritePackageLock(source), expected); assert.equal(rewritePackageLock(expected), expected); assert.equal(normalizePackageLock(expected), source.replace('https://registry.yarnpkg.com/', publicRegistry)); }); +test('normalizes slashless registry overrides before rewriting resolved URLs', () => { + for (const registry of [publicRegistry, microsoftRegistry]) { + const expected = source.replaceAll(publicRegistry, registry).replace('https://registry.yarnpkg.com/', registry); + assert.equal(rewritePackageLock(source, registry.slice(0, -1)), expected); + } +}); + test('supports version 2 legacy dependencies as well as packages', () => { const legacy = JSON.stringify({ lockfileVersion: 2, packages: {}, dependencies: { example: { resolved: `${publicRegistry}example/-/example.tgz` } }, }); - assert.equal(rewritePackageLock(legacy), legacy.replace(publicRegistry, proxyRegistry)); + assert.equal(rewritePackageLock(legacy), legacy.replace(publicRegistry, microsoftRegistry)); }); test('preserves other fields, private feeds, Git dependencies and lookalike hosts', () => { @@ -100,12 +107,12 @@ test('missing or invalid root lockfile fails explicitly', (t) => { assert.throws(() => rewritePackageLocks(root), /Expected an npm lockfile/); }); -function runCli({ args = ['--ci'], npmCli = 'npm-cli.js', result = { status: 0 } } = {}) { +function runCli({ args = ['--ci'], npmCli = 'npm-cli.js', result = { status: 0 }, env = {} } = {}) { const calls = []; const errors = []; const process = { argv: ['node', 'setup-npm-registry.js', ...args], - env: { npm_execpath: npmCli }, + env: { npm_execpath: npmCli, ...env }, execPath: 'node', }; const module = { exports: {} }; @@ -113,7 +120,7 @@ function runCli({ args = ['--ci'], npmCli = 'npm-cli.js', result = { status: 0 } if (name === 'node:fs') { return { readFileSync: () => source, - writeFileSync: (file) => calls.push({ file }), + writeFileSync: (file, source) => calls.push({ file, source }), }; } if (name === 'node:child_process') { @@ -137,12 +144,12 @@ function runCli({ args = ['--ci'], npmCli = 'npm-cli.js', result = { status: 0 } return { calls, errors, process }; } -test('ci:msft rewrites the root lockfile before installing with the shared proxy registry', () => { +test('ci:msft rewrites the root lockfile before installing with the Microsoft registry', () => { const context = runCli(); assert.equal(context.calls.length, 2); assert.ok(context.calls[0].file.endsWith('package-lock.json')); assert.equal(context.calls[1].command, 'node'); - assert.deepEqual(context.calls[1].args, ['npm-cli.js', 'ci', `--registry=${proxyRegistry}`]); + assert.deepEqual(context.calls[1].args, ['npm-cli.js', 'ci', `--registry=${microsoftRegistry}`]); assert.equal(context.calls[1].cwd, path.resolve(__dirname, '..')); assert.equal(context.calls[1].stdio, 'inherit'); assert.equal(context.errors.length, 0); @@ -150,6 +157,33 @@ test('ci:msft rewrites the root lockfile before installing with the shared proxy assert.equal(scripts['ci:msft'], 'node ./build/setup-npm-registry.js --ci'); }); +test('ci:msft preserves template authentication for the configured internal registry', () => { + const registry = 'https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/'; + for (const settings of [ + { NPM_CONFIG_REGISTRY: registry }, + { npm_config_registry: registry }, + { NPM_CONFIG_REGISTRY: registry.slice(0, -1) }, + { npm_config_registry: registry.slice(0, -1) }, + { NPM_CONFIG_REGISTRY: 'https://stale.example/npm/', npm_config_registry: registry }, + ]) { + const env = { + ...settings, + NPM_CONFIG_USERCONFIG: path.join(tmpdir(), 'authenticated.npmrc'), + NODE_AUTH_TOKEN: 'test-token', + }; + const context = runCli({ env }); + const install = context.calls.at(-1); + assert.deepEqual(install.args, ['npm-cli.js', 'ci', `--registry=${registry}`]); + assert.equal(install.env, undefined); + assert.equal(context.process.env.NPM_CONFIG_USERCONFIG, env.NPM_CONFIG_USERCONFIG); + assert.equal(context.process.env.NODE_AUTH_TOKEN, env.NODE_AUTH_TOKEN); + for (const write of context.calls.slice(0, -1)) { + assert.equal(write.source, rewritePackageLock(source, registry)); + } + assert.equal(context.errors.length, 0); + } +}); + for (const result of [{ status: 7 }, { status: null, signal: 'SIGTERM' }, { error: new Error('not found') }]) { test(`ci:msft reports npm failure: ${result.signal || result.error?.message || result.status}`, () => { const context = runCli({ result }); diff --git a/build/setup.test.js b/build/setup.test.js index e75ac2f6..664c0e6a 100644 --- a/build/setup.test.js +++ b/build/setup.test.js @@ -90,11 +90,15 @@ for (const platform of ['win32', 'linux']) { for (const call of calls) { assert.equal( call.env.npm_config_registry, - microsoft ? 'https://packagefeedproxy.microsoft.io/npm/' : 'https://registry.npmjs.org/', + microsoft + ? 'https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/Pylance_PublicPackages/npm/registry/' + : 'https://registry.npmjs.org/', ); assert.equal( call.env.PIP_INDEX_URL, - microsoft ? 'https://packagefeedproxy.microsoft.io/pypi/simple/' : 'https://pypi.org/simple/', + microsoft + ? 'https://pkgs.dev.azure.com/devdiv/DevDiv/_packaging/debugpy/pypi/simple/' + : 'https://pypi.org/simple/', ); assert.equal(call.env.NPM_CONFIG_REGISTRY, undefined); assert.equal(call.env.HTTPS_PROXY, context.originalEnv.HTTPS_PROXY); @@ -180,10 +184,11 @@ test('Microsoft entry point selects the shared Microsoft setup implementation', test('PR and push installs use the npm helper without a redundant registry environment setting', () => { const root = path.join(__dirname, '..'); + const { microsoft } = require('./package-feeds.json'); for (const file of ['pr-check.yml', 'push-check.yml']) { const workflow = readFileSync(path.join(root, '.github', 'workflows', file), 'utf8'); assert.doesNotMatch(workflow, /^\s*NPM_CONFIG_REGISTRY:/m); - assert.match(workflow, /^\s*PIP_INDEX_URL: https:\/\/packagefeedproxy\.microsoft\.io\/pypi\/simple\//m); + assert.ok(workflow.split(/\r?\n/).includes(` PIP_INDEX_URL: ${microsoft.pip}`)); assert.doesNotMatch(workflow, /^\s*run: npm ci\b/m); } for (const action of ['lint', 'build-vsix']) { @@ -191,5 +196,6 @@ test('PR and push installs use the npm helper without a redundant registry envir assert.match(workflow, /^\s*run: npm run ci:msft\s*$/m); } const codeql = readFileSync(path.join(root, '.github', 'workflows', 'codeql-analysis.yml'), 'utf8'); - assert.match(codeql, /^\s*NPM_CONFIG_REGISTRY: https:\/\/packagefeedproxy\.microsoft\.io\/npm\//m); + assert.ok(codeql.split(/\r?\n/).includes(` NPM_CONFIG_REGISTRY: ${microsoft.npm}`)); + assert.ok(codeql.split(/\r?\n/).includes(` PIP_INDEX_URL: ${microsoft.pip}`)); });