From 96f1046406005dd7449ef0206ce499aaaaa449f3 Mon Sep 17 00:00:00 2001 From: Jan Kowalleck Date: Thu, 31 Oct 2024 10:48:55 +0100 Subject: [PATCH] tests: restructure integration tests (#1231) - tests `from-setup` were disabled due to inconsistencies between npm6,7,8,9 they were a backup an actually not needed, as they are backed up by the demo data that was built from them. - tests `edge-cases`, `dogfooding` and `args-pass-through` were refactored to own files - `args-pass-through` was enhanced --------- Signed-off-by: Jan Kowalleck --- .github/workflows/nodejs.yml | 2 + .github/workflows/npm-ls_demo-results.yml | 4 +- demo/local-dependencies/project/.gitignore | 2 +- demo/local-dependencies/project/.npmrc | 2 + jest.config.js | 9 +- tests/_helper/index.js | 4 +- .../integration/cli.args-pass-through.test.js | 93 ++++++ tests/integration/cli.dogfooding.test.js | 49 +++ tests/integration/cli.edge-cases.test.js | 165 ++++++++++ tests/integration/cli.from-collected.test.js | 281 +----------------- tests/integration/cli.from-setups.test.js | 27 +- tests/integration/index.js | 114 +++++++ tests/integration/setup.js | 30 +- 13 files changed, 469 insertions(+), 313 deletions(-) create mode 100644 tests/integration/cli.args-pass-through.test.js create mode 100644 tests/integration/cli.dogfooding.test.js create mode 100644 tests/integration/cli.edge-cases.test.js create mode 100644 tests/integration/index.js diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index fe501dea..e6ea1850 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -102,6 +102,8 @@ jobs: with: node-version: ${{ matrix.node-version }} # cache: 'npm' + - name: npm version + run: npm --version - name: setup subject shell: bash run: | diff --git a/.github/workflows/npm-ls_demo-results.yml b/.github/workflows/npm-ls_demo-results.yml index c0a0c439..9d806dbb 100644 --- a/.github/workflows/npm-ls_demo-results.yml +++ b/.github/workflows/npm-ls_demo-results.yml @@ -51,12 +51,12 @@ jobs: - macos-latest include: - subject: local-workspaces - additional_npm-ls_args: '-w my-local-e' + additional_npm-ls_args: '--workspace==my-local-e' npm-version: '10' # Current node-version: '22' # Current os: ubuntu-latest - subject: local-workspaces - additional_npm-ls_args: '-w my-local -w my-local-e' + additional_npm-ls_args: '--workspace==my-local --workspace==my-local-e' npm-version: '10' # Current node-version: '22' # Current os: ubuntu-latest diff --git a/demo/local-dependencies/project/.gitignore b/demo/local-dependencies/project/.gitignore index a743821c..89a5d10b 100644 --- a/demo/local-dependencies/project/.gitignore +++ b/demo/local-dependencies/project/.gitignore @@ -1,7 +1,7 @@ /* !/.gitignore -!/package.json !/.npmrc +!/package.json !/README.md !/packages/ !/packages/** diff --git a/demo/local-dependencies/project/.npmrc b/demo/local-dependencies/project/.npmrc index 2b756953..687abd56 100644 --- a/demo/local-dependencies/project/.npmrc +++ b/demo/local-dependencies/project/.npmrc @@ -1,2 +1,4 @@ +; see the docs: https://docs.npmjs.com/cli/v7/using-npm/config + # mitigate https://github.com/npm/cli/issues/5733 install-links=false diff --git a/jest.config.js b/jest.config.js index 7e5f4de5..b553f0ee 100644 --- a/jest.config.js +++ b/jest.config.js @@ -186,9 +186,12 @@ module.exports = { ], // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped - // testPathIgnorePatterns: [ - // "/node_modules/" - // ], + testPathIgnorePatterns: [ + '/node_modules/', + '/_data/', + '/_helper/', + '/_tmp/' + ], // The regexp pattern or array of patterns that Jest uses to detect test files // testRegex: [], diff --git a/tests/_helper/index.js b/tests/_helper/index.js index 835c5476..cd01ed64 100644 --- a/tests/_helper/index.js +++ b/tests/_helper/index.js @@ -146,11 +146,13 @@ function makeXmlReproducible (xml) { * @return {number[]} */ function getNpmVersion () { - return spawnSync('npm', ['--version'], { + const v = spawnSync('npm', ['--version'], { stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf8', shell: process.platform.startsWith('win') }).stdout.split('.').map(Number) + process.stderr.write(`\ndetected npm version: ${JSON.stringify(v)}\n`) + return v } module.exports = { diff --git a/tests/integration/cli.args-pass-through.test.js b/tests/integration/cli.args-pass-through.test.js new file mode 100644 index 00000000..f9cc7521 --- /dev/null +++ b/tests/integration/cli.args-pass-through.test.js @@ -0,0 +1,93 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { join } = require('path') +const { mkdirSync, readFileSync } = require('fs') + +const { describe, expect, test } = require('@jest/globals') + +const { mkTemp, runCLI, dummyProjectsRoot, npmLsReplacement } = require('./') + +describe('integration.cli.args-pass-through', () => { + const cliRunTestTimeout = 15000 + + const tmpRoot = mkTemp('cli.args-pass-through') + + describe('npm-version depending npm-args', () => { + const tmpRootRun = join(tmpRoot, 'npmVersion-depending-npmArgs') + mkdirSync(tmpRootRun) + + const rMinor = Math.round(99 * Math.random()) + const rPatch = Math.round(99 * Math.random()) + const le6 = Math.round(6 * Math.random()) + const ge7 = 7 + Math.round(92 * Math.random()) + + const npmArgsGeneral = ['--json', '--long'] + const npm6ArgsGeneral = [...npmArgsGeneral, '--depth=255'] + const npm7ArgsGeneral = [...npmArgsGeneral, '--all'] + const npm8ArgsGeneral = [...npmArgsGeneral, '--all'] + const npm9ArgsGeneral = [...npmArgsGeneral, '--all'] + const npm10ArgsGeneral = [...npmArgsGeneral, '--all'] + + test.each([ + // region basic + ['basic npm 6', `6.${rMinor}.${rPatch}`, [], npm6ArgsGeneral], + ['basic npm 7', `7.${rMinor}.${rPatch}`, [], npm7ArgsGeneral], + ['basic npm 8', `8.${rMinor}.${rPatch}`, [], npm8ArgsGeneral], + ['basic npm 9', `9.${rMinor}.${rPatch}`, [], npm9ArgsGeneral], + ['basic npm 10', `10.${rMinor}.${rPatch}`, [], npm10ArgsGeneral], + // endregion basic + // region omit + ['omit everything npm 6', `6.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm6ArgsGeneral, '--production']], + ['omit everything npm 7', `7.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm7ArgsGeneral, '--production']], + ['omit everything npm lower 8.7', `8.${le6}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm8ArgsGeneral, '--production']], + ['omit everything npm greater-equal 8.7', `8.${ge7}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm8ArgsGeneral, '--omit=dev', '--omit=optional', '--omit=peer']], + ['omit everything npm 9', `9.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm9ArgsGeneral, '--omit=dev', '--omit=optional', '--omit=peer']], + ['omit everything npm 10', `10.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm10ArgsGeneral, '--omit=dev', '--omit=optional', '--omit=peer']], + // endregion omit + // region package-lock-only + ['package-lock-only not supported npm 6 ', `6.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm6ArgsGeneral]], + ['package-lock-only npm 7', `7.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm7ArgsGeneral, '--package-lock-only']], + ['package-lock-only npm 8', `8.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm8ArgsGeneral, '--package-lock-only']], + ['package-lock-only npm 9', `9.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm9ArgsGeneral, '--package-lock-only']], + ['package-lock-only npm 10', `10.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm10ArgsGeneral, '--package-lock-only']] + // endregion package-lock-only + ])('%s', async (purpose, npmVersion, cdxArgs, expectedArgs) => { + const logFileBase = join(tmpRootRun, purpose.replace(/\W/g, '_')) + const cwd = dummyProjectsRoot + + const { res, errFile } = runCLI([ + ...cdxArgs, + '--', + join('with-lockfile', 'package.json') + ], logFileBase, cwd, { + CT_VERSION: npmVersion, + CT_EXPECTED_ARGS: expectedArgs.join(' '), + npm_execpath: npmLsReplacement.checkArgs + }) + + try { + await expect(res).resolves.toBe(0) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + }, cliRunTestTimeout) + }) +}) diff --git a/tests/integration/cli.dogfooding.test.js b/tests/integration/cli.dogfooding.test.js new file mode 100644 index 00000000..d758e994 --- /dev/null +++ b/tests/integration/cli.dogfooding.test.js @@ -0,0 +1,49 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { spawnSync } = require('child_process') + +const { describe, expect, test } = require('@jest/globals') + +const { projectRootPath, cliWrapper } = require('./') + +describe('integration.cli.dogfooding', () => { + const cliRunTestTimeout = 15000 + + test.each(['JSON', 'XML'])('dogfooding %s', (format) => { + const res = spawnSync( + process.execPath, + ['--', cliWrapper, '--output-format', format, '--ignore-npm-errors'], + { + cwd: projectRootPath, + stdio: ['ignore', 'inherit', 'pipe'], + encoding: 'utf8' + } + ) + try { + expect(res.error).toBeUndefined() + expect(res.status).toBe(0) + } catch (err) { + process.stderr.write('\n') + process.stderr.write(res.stderr) + process.stderr.write('\n') + throw err + } + }, cliRunTestTimeout) +}) diff --git a/tests/integration/cli.edge-cases.test.js b/tests/integration/cli.edge-cases.test.js new file mode 100644 index 00000000..63a1f247 --- /dev/null +++ b/tests/integration/cli.edge-cases.test.js @@ -0,0 +1,165 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { join } = require('path') +const { mkdirSync, writeFileSync, readFileSync } = require('fs') + +const { describe, expect, test } = require('@jest/globals') + +const { makeReproducible } = require('../_helper') +const { UPDATE_SNAPSHOTS, mkTemp, runCLI, latestCdxSpecVersion, dummyProjectsRoot, npmLsReplacement, demoResultsRoot } = require('./') + +describe('integration.cli.edge-cases', () => { + const cliRunTestTimeout = 15000 + + const tmpRoot = mkTemp('cli.edge_cases') + + describe('broken project', () => { + const tmpRootRun = join(tmpRoot, 'broken-project') + mkdirSync(tmpRootRun) + + test.each([ + ['no-lockfile', /missing evidence/i], + ['no-manifest', /missing .*manifest file/i] + ])('%s', async (folderName, expectedError) => { + const logFileBase = join(tmpRootRun, folderName) + const cwd = join(dummyProjectsRoot, folderName) + + const { res, errFile } = runCLI([], logFileBase, cwd, { npm_execpath: undefined }) + + try { + await expect(res).rejects.toThrow(expectedError) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + }, cliRunTestTimeout) + }) + + describe('with broken npm-ls', () => { + const tmpRootRun = join(tmpRoot, 'with-broken') + mkdirSync(tmpRootRun) + + test('error on non-existing binary', async () => { + const logFileBase = join(tmpRootRun, 'non-existing') + const cwd = join(dummyProjectsRoot, 'with-lockfile') + + const { res, errFile } = runCLI([], logFileBase, cwd, { + npm_execpath: npmLsReplacement.nonExistingBinary + }) + + try { + await expect(res).rejects.toThrow(/^unexpected npm execpath/i) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + }, cliRunTestTimeout) + + test('error on non-zero exit', async () => { + const logFileBase = join(tmpRootRun, 'error-exit-nonzero') + const cwd = join(dummyProjectsRoot, 'with-lockfile') + + const expectedExitCode = 1 + Math.floor(254 * Math.random()) + + const { res, errFile } = runCLI([], logFileBase, cwd, { + CT_VERSION: '8.99.0', + // non-zero exit code + CT_EXIT_CODE: `${expectedExitCode}`, + npm_execpath: npmLsReplacement.justExit + }) + + try { + await expect(res).rejects.toThrow(`npm-ls exited with errors: ${expectedExitCode} noSignal`) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + }, cliRunTestTimeout) + + test('error on broken json response', async () => { + const logFileBase = join(tmpRootRun, 'error-json-broken') + const cwd = join(dummyProjectsRoot, 'with-lockfile') + + const { res, errFile } = runCLI([], logFileBase, cwd, { + CT_VERSION: '8.99.0', + // abuse the npm-ls replacement, as it can be caused to crash under control. + npm_execpath: npmLsReplacement.brokenJson + }) + + try { + await expect(res).rejects.toThrow(/failed to parse npm-ls response/i) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + }, cliRunTestTimeout) + }) + + test('suppressed error on non-zero exit', async () => { + const dd = { subject: 'dev-dependencies', npm: '8', node: '14', os: 'ubuntu-latest' } + + mkdirSync(join(tmpRoot, 'suppressed-error-on-non-zero-exit')) + const expectedOutSnap = join(demoResultsRoot, 'suppressed-error-on-non-zero-exit', `${dd.subject}_npm${dd.npm}_node${dd.node}_${dd.os}.snap.json`) + const logFileBase = join(tmpRoot, 'suppressed-error-on-non-zero-exit', `${dd.subject}_npm${dd.npm}_node${dd.node}_${dd.os}`) + const cwd = dummyProjectsRoot + + const expectedExitCode = 1 + Math.floor(254 * Math.random()) + + const { res, outFile, errFile } = runCLI([ + '-vvv', + '--ignore-npm-errors', + '--output-reproducible', + // no intention to test all the spec-versions nor all the output-formats - this would be not our scope. + '--spec-version', `${latestCdxSpecVersion}`, + '--output-format', 'JSON', + // prevent file interaction in this synthetic scenario - they would not exist anyway + '--package-lock-only', + '--', + join('with-lockfile', 'package.json') + ], logFileBase, cwd, { + CT_VERSION: `${dd.npm}.99.0`, + // non-zero exit code + CT_EXIT_CODE: expectedExitCode, + CT_SUBJECT: dd.subject, + CT_NPM: dd.npm, + CT_NODE: dd.node, + CT_OS: dd.os, + npm_execpath: npmLsReplacement.demoResults + }) + + try { + await expect(res).resolves.toBe(0) + } catch (err) { + process.stderr.write(readFileSync(errFile)) + throw err + } + + const actualOutput = makeReproducible('json', readFileSync(outFile, 'utf8')) + + if (UPDATE_SNAPSHOTS) { + writeFileSync(expectedOutSnap, actualOutput, 'utf8') + } + + expect(actualOutput).toEqual( + readFileSync(expectedOutSnap, 'utf8'), + `${outFile} should equal ${expectedOutSnap}` + ) + }, cliRunTestTimeout) +}) diff --git a/tests/integration/cli.from-collected.test.js b/tests/integration/cli.from-collected.test.js index 109bdb32..fa6c24b3 100644 --- a/tests/integration/cli.from-collected.test.js +++ b/tests/integration/cli.from-collected.test.js @@ -17,164 +17,19 @@ SPDX-License-Identifier: Apache-2.0 Copyright (c) OWASP Foundation. All Rights Reserved. */ -const { spawnSync } = require('child_process') -const { resolve, join } = require('path') -const { mkdtempSync, mkdirSync, createWriteStream, openSync, writeFileSync, readFileSync } = require('fs') +const { join } = require('path') +const { mkdirSync, writeFileSync, readFileSync } = require('fs') -const { Spec } = require('@cyclonedx/cyclonedx-library') const { describe, expect, test } = require('@jest/globals') const { index: indexNpmLsDemoData } = require('../_data/npm-ls_demo-results') -const cli = require('../../dist/cli') const { makeReproducible } = require('../_helper') - -const projectRootPath = resolve(__dirname, '..', '..') -const projectTestRootPath = resolve(__dirname, '..') - -const cliWrapper = join(projectRootPath, 'bin', 'cyclonedx-npm-cli.js') - -/* we run only the latest most advanced */ -const latestCdxSpecVersion = Spec.Version.v1dot6 +const { UPDATE_SNAPSHOTS, mkTemp, dummyProjectsRoot, runCLI, latestCdxSpecVersion, demoResultsRoot, npmLsReplacement } = require('./') describe('integration.cli.from-collected', () => { - const UPDATE_SNAPSHOTS = !!process.env.CNPM_TEST_UPDATE_SNAPSHOTS const cliRunTestTimeout = 15000 - const tmpRoot = mkdtempSync(join(projectTestRootPath, '_tmp', 'CDX-IT-cli.from-collected.')) - - const dummyProjectsRoot = resolve(projectTestRootPath, '_data', 'dummy_projects') - const demoResultsRoot = resolve(projectTestRootPath, '_data', 'sbom_demo-results') - const npmLsReplacementPath = resolve(projectTestRootPath, '_data', 'npm-ls_replacement') - - const npmLsReplacement = { - brokenJson: resolve(npmLsReplacementPath, 'broken-json.js'), - checkArgs: resolve(npmLsReplacementPath, 'check-args.js'), - demoResults: resolve(npmLsReplacementPath, 'demo-results.js'), - justExit: resolve(npmLsReplacementPath, 'just-exit.js'), - nonExistingBinary: resolve(npmLsReplacementPath, 'aNonExistingBinary') - } - - /** - * @param {string[]} args - * @param {string} logFileBase - * @param {string} cwd - * @param {Object.} env - * @return {{res: Promise., outFile:string, errFile:string}} - */ - function runCLI (args, logFileBase, cwd, env) { - const outFile = `${logFileBase}.out` - const outFD = openSync(outFile, 'w') - const stdout = createWriteStream(null, { fd: outFD }) - - const errFile = `${logFileBase}.err` - const errFD = openSync(errFile, 'w') - const stderr = createWriteStream(null, { fd: errFD }) - - /** @type Partial */ - const mockProcess = { - stdout, - stderr, - cwd: () => cwd, - execPath: process.execPath, - argv0: process.argv0, - argv: [ - process.argv[0], - 'dummy_process', - ...args - ], - env: { - ...process.env, - ...env - } - } - - /** @type Promise. */ - const res = cli.run(mockProcess) - - return { res, outFile, errFile } - } - - describe('broken project', () => { - const tmpRootRun = join(tmpRoot, 'broken-project') - mkdirSync(tmpRootRun) - - test.each([ - ['no-lockfile', /missing evidence/i], - ['no-manifest', /missing .*manifest file/i] - ])('%s', async (folderName, expectedError) => { - const logFileBase = join(tmpRootRun, folderName) - const cwd = resolve(dummyProjectsRoot, folderName) - - const { res, errFile } = runCLI([], logFileBase, cwd, { npm_execpath: undefined }) - - try { - await expect(res).rejects.toThrow(expectedError) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - }, cliRunTestTimeout) - }) - - describe('with broken npm-ls', () => { - const tmpRootRun = join(tmpRoot, 'with-broken') - mkdirSync(tmpRootRun) - - test('error on non-existing binary', async () => { - const logFileBase = join(tmpRootRun, 'non-existing') - const cwd = resolve(dummyProjectsRoot, 'with-lockfile') - - const { res, errFile } = runCLI([], logFileBase, cwd, { - npm_execpath: npmLsReplacement.nonExistingBinary - }) - - try { - await expect(res).rejects.toThrow(/^unexpected npm execpath/i) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - }, cliRunTestTimeout) - - test('error on non-zero exit', async () => { - const logFileBase = join(tmpRootRun, 'error-exit-nonzero') - const cwd = resolve(dummyProjectsRoot, 'with-lockfile') - - const expectedExitCode = 1 + Math.floor(254 * Math.random()) - - const { res, errFile } = runCLI([], logFileBase, cwd, { - CT_VERSION: '8.99.0', - // non-zero exit code - CT_EXIT_CODE: `${expectedExitCode}`, - npm_execpath: npmLsReplacement.justExit - }) - - try { - await expect(res).rejects.toThrow(`npm-ls exited with errors: ${expectedExitCode} noSignal`) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - }, cliRunTestTimeout) - - test('error on broken json response', async () => { - const logFileBase = join(tmpRootRun, 'error-json-broken') - const cwd = resolve(dummyProjectsRoot, 'with-lockfile') - - const { res, errFile } = runCLI([], logFileBase, cwd, { - CT_VERSION: '8.99.0', - // abuse the npm-ls replacement, as it can be caused to crash under control. - npm_execpath: npmLsReplacement.brokenJson - }) - - try { - await expect(res).rejects.toThrow(/failed to parse npm-ls response/i) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - }, cliRunTestTimeout) - }) + const tmpRoot = mkTemp('cli.from-collected') describe('with prepared npm-ls', () => { const tmpRootRun = join(tmpRoot, 'with-prepared') @@ -189,9 +44,9 @@ describe('integration.cli.from-collected', () => { mkdirSync(join(tmpRootRun, ud.subject)) test.each(demoCases)('$subject $args npm$npm node$node $os', async (dd) => { - const expectedOutSnap = resolve(demoResultsRoot, ud.subject, `${dd.subject}${dd.args}_npm${dd.npm}_node${dd.node}_${dd.os}.snap.json`) + const expectedOutSnap = join(demoResultsRoot, ud.subject, `${dd.subject}${dd.args}_npm${dd.npm}_node${dd.node}_${dd.os}.snap.json`) const logFileBase = join(tmpRootRun, ud.subject, `${dd.subject}${dd.args}_npm${dd.npm}_node${dd.node}_${dd.os}`) - const cwd = resolve(projectTestRootPath, '_data', 'dummy_projects') + const cwd = dummyProjectsRoot const { res, outFile, errFile } = runCLI([ '-vvv', @@ -240,128 +95,4 @@ describe('integration.cli.from-collected', () => { }, cliRunTestTimeout) }) }) - - test('suppressed error on non-zero exit', async () => { - const dd = { subject: 'dev-dependencies', npm: '8', node: '14', os: 'ubuntu-latest' } - - mkdirSync(join(tmpRoot, 'suppressed-error-on-non-zero-exit')) - const expectedOutSnap = resolve(demoResultsRoot, 'suppressed-error-on-non-zero-exit', `${dd.subject}_npm${dd.npm}_node${dd.node}_${dd.os}.snap.json`) - const logFileBase = join(tmpRoot, 'suppressed-error-on-non-zero-exit', `${dd.subject}_npm${dd.npm}_node${dd.node}_${dd.os}`) - const cwd = resolve(projectTestRootPath, '_data', 'dummy_projects') - - const expectedExitCode = 1 + Math.floor(254 * Math.random()) - - const { res, outFile, errFile } = runCLI([ - '-vvv', - '--ignore-npm-errors', - '--output-reproducible', - // no intention to test all the spec-versions nor all the output-formats - this would be not our scope. - '--spec-version', `${latestCdxSpecVersion}`, - '--output-format', 'JSON', - // prevent file interaction in this synthetic scenario - they would not exist anyway - '--package-lock-only', - '--', - join('with-lockfile', 'package.json') - ], logFileBase, cwd, { - CT_VERSION: `${dd.npm}.99.0`, - // non-zero exit code - CT_EXIT_CODE: expectedExitCode, - CT_SUBJECT: dd.subject, - CT_NPM: dd.npm, - CT_NODE: dd.node, - CT_OS: dd.os, - npm_execpath: npmLsReplacement.demoResults - }) - - try { - await expect(res).resolves.toBe(0) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - - const actualOutput = makeReproducible('json', readFileSync(outFile, 'utf8')) - - if (UPDATE_SNAPSHOTS) { - writeFileSync(expectedOutSnap, actualOutput, 'utf8') - } - - expect(actualOutput).toEqual( - readFileSync(expectedOutSnap, 'utf8'), - `${outFile} should equal ${expectedOutSnap}` - ) - }, cliRunTestTimeout) - - describe('npm-version depending npm-args', () => { - const tmpRootRun = join(tmpRoot, 'npmVersion-depending-npmArgs') - mkdirSync(tmpRootRun) - - const rMinor = Math.round(99 * Math.random()) - const rPatch = Math.round(99 * Math.random()) - const le6 = Math.round(6 * Math.random()) - const ge7 = 7 + Math.round(92 * Math.random()) - - const npmArgsGeneral = ['--json', '--long'] - const npm6ArgsGeneral = [...npmArgsGeneral, '--depth=255'] - const npm7ArgsGeneral = [...npmArgsGeneral, '--all'] - const npm8ArgsGeneral = [...npmArgsGeneral, '--all'] - - test.each([ - ['basic npm 6', `6.${rMinor}.${rPatch}`, [], npm6ArgsGeneral], - ['basic npm 7', `7.${rMinor}.${rPatch}`, [], npm7ArgsGeneral], - ['basic npm 8', `8.${rMinor}.${rPatch}`, [], npm8ArgsGeneral], - // region omit - ['omit everything npm 6', `6.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm6ArgsGeneral, '--production']], - ['omit everything npm 7', `7.${rMinor}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm7ArgsGeneral, '--production']], - ['omit everything npm lower 8.7', `8.${le6}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm8ArgsGeneral, '--production']], - ['omit everything npm greater-equal 8.7 ', `8.${ge7}.${rPatch}`, ['--omit', 'dev', 'optional', 'peer'], [...npm8ArgsGeneral, '--omit=dev', '--omit=optional', '--omit=peer']], - // endregion - // region package-lock-only - ['package-lock-only not supported npm 6 ', `6.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm6ArgsGeneral]], - ['package-lock-only npm 7', `7.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm7ArgsGeneral, '--package-lock-only']], - ['package-lock-only npm 8', `8.${rMinor}.${rPatch}`, ['--package-lock-only'], [...npm8ArgsGeneral, '--package-lock-only']] - // endregion - ])('%s', async (purpose, npmVersion, cdxArgs, expectedArgs) => { - const logFileBase = join(tmpRootRun, purpose.replace(/\W/g, '_')) - const cwd = resolve(projectTestRootPath, '_data', 'dummy_projects') - - const { res, errFile } = runCLI([ - ...cdxArgs, - '--', - join('with-lockfile', 'package.json') - ], logFileBase, cwd, { - CT_VERSION: npmVersion, - CT_EXPECTED_ARGS: expectedArgs.join(' '), - npm_execpath: npmLsReplacement.checkArgs - }) - - try { - await expect(res).resolves.toBe(0) - } catch (err) { - process.stderr.write(readFileSync(errFile)) - throw err - } - }, cliRunTestTimeout) - }) - - test.each(['JSON', 'XML'])('dogfooding %s', (format) => { - const res = spawnSync( - process.execPath, - ['--', cliWrapper, '--output-format', format, '--ignore-npm-errors'], - { - cwd: projectRootPath, - stdio: ['ignore', 'inherit', 'pipe'], - encoding: 'utf8' - } - ) - try { - expect(res.error).toBeUndefined() - expect(res.status).toBe(0) - } catch (err) { - process.stderr.write('\n') - process.stderr.write(res.stderr) - process.stderr.write('\n') - throw err - } - }, cliRunTestTimeout) }) diff --git a/tests/integration/cli.from-setups.test.js b/tests/integration/cli.from-setups.test.js index bc2a5a86..3e7d1ba6 100644 --- a/tests/integration/cli.from-setups.test.js +++ b/tests/integration/cli.from-setups.test.js @@ -18,30 +18,23 @@ Copyright (c) OWASP Foundation. All Rights Reserved. */ const { spawnSync } = require('child_process') -const { resolve, join } = require('path') -const { mkdtempSync, writeFileSync, readFileSync } = require('fs') +const { join } = require('path') +const { writeFileSync, readFileSync } = require('fs') const { describe, expect, test } = require('@jest/globals') -const { Spec } = require('@cyclonedx/cyclonedx-library') const { makeReproducible, getNpmVersion } = require('../_helper') +const { UPDATE_SNAPSHOTS, mkTemp, cliWrapper, latestCdxSpecVersion, demoResultsRoot, projectDemoRootPath } = require('./') -const projectRootPath = resolve(__dirname, '..', '..') -const projectTestRootPath = join(projectRootPath, 'tests') -const demoRootPath = join(projectRootPath, 'demo') - -const cliWrapper = join(projectRootPath, 'bin', 'cyclonedx-npm-cli.js') - -/* we run only the latest most advanced */ -const latestCdxSpecVersion = Spec.Version.v1dot6 - -describe('integration.cli.from-setups', () => { +// skipped for now +describe.skip('integration.cli.from-setups', () => { + // !! due to inconsistencies between npm6,7,8 - + // some test beds might be skipped const skipAllTests = getNpmVersion()[0] < 8 - const UPDATE_SNAPSHOTS = 1 // !!process.env.CNPM_TEST_UPDATE_SNAPSHOTS const cliRunTestTimeout = 15000 - const tmpRoot = mkdtempSync(join(projectTestRootPath, '_tmp', 'CDX-IT-cli.from-setups.')) + const tmpRoot = mkTemp('cli.from-setups') const demos = [ 'alternative-package-registry', @@ -56,8 +49,6 @@ describe('integration.cli.from-setups', () => { ] const formats = ['json', 'xml'] - const demoResultsRoot = resolve(projectTestRootPath, '_data', 'sbom_demo-results') - /** * @param {string} demo * @param {string} oType @@ -77,7 +68,7 @@ describe('integration.cli.from-setups', () => { '--output-file', outFile, '--validate' ], { - cwd: join(demoRootPath, demo, 'project'), + cwd: join(projectDemoRootPath, demo, 'project'), stdio: ['ignore', 'inherit', 'pipe'], encoding: 'utf8' } diff --git a/tests/integration/index.js b/tests/integration/index.js new file mode 100644 index 00000000..1ea5d34d --- /dev/null +++ b/tests/integration/index.js @@ -0,0 +1,114 @@ +/*! +This file is part of CycloneDX generator for NPM projects. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +Copyright (c) OWASP Foundation. All Rights Reserved. +*/ + +const { Spec } = require('@cyclonedx/cyclonedx-library') +const { mkdtempSync } = require('fs') +const { join, resolve } = require('path') + +const { createWriteStream, openSync } = require('fs') + +const cli = require('../../dist/cli') + +const projectRootPath = resolve(__dirname, '..', '..') +const projectTestRootPath = resolve(__dirname, '..') + +const projectDemoRootPath = join(projectRootPath, 'demo') +const projectTestDataPath = join(projectTestRootPath, '_data') + +const dummyProjectsRoot = join(projectTestDataPath, 'dummy_projects') +const demoResultsRoot = join(projectTestDataPath, 'sbom_demo-results') +const npmLsReplacementPath = join(projectTestDataPath, 'npm-ls_replacement') + +const npmLsReplacement = { + brokenJson: join(npmLsReplacementPath, 'broken-json.js'), + checkArgs: join(npmLsReplacementPath, 'check-args.js'), + demoResults: join(npmLsReplacementPath, 'demo-results.js'), + justExit: join(npmLsReplacementPath, 'just-exit.js'), + nonExistingBinary: join(npmLsReplacementPath, 'aNonExistingBinary') +} + +/* we might run only the latest most advanced */ +const latestCdxSpecVersion = Spec.Version.v1dot6 + +const UPDATE_SNAPSHOTS = !!process.env.CNPM_TEST_UPDATE_SNAPSHOTS + +/** + * @param {string[]} args + * @param {string} logFileBase + * @param {string} cwd + * @param {Object.} env + * @return {{res: Promise., outFile:string, errFile:string}} + */ +function runCLI (args, logFileBase, cwd, env) { + const outFile = `${logFileBase}.out` + const outFD = openSync(outFile, 'w') + const stdout = createWriteStream(null, { fd: outFD }) + + const errFile = `${logFileBase}.err` + const errFD = openSync(errFile, 'w') + const stderr = createWriteStream(null, { fd: errFD }) + + /** @type Partial */ + const mockProcess = { + stdout, + stderr, + cwd: () => cwd, + execPath: process.execPath, + argv0: process.argv0, + argv: [ + process.argv[0], + 'dummy_process', + ...args + ], + env: { + ...process.env, + ...env + } + } + + /** @type Promise. */ + const res = cli.run(mockProcess) + + return { res, outFile, errFile } +} + +const cliWrapper = join(projectRootPath, 'bin', 'cyclonedx-npm-cli.js') + +/** + * @param {string} caseName + * @return {string} + */ +function mkTemp (caseName) { + return mkdtempSync(join(projectTestRootPath, '_tmp', `CDX-IT-${caseName}.`)) +} + +module.exports = { + UPDATE_SNAPSHOTS, + latestCdxSpecVersion, + projectRootPath, + projectDemoRootPath, + projectTestDataPath, + demoResultsRoot, + dummyProjectsRoot, + npmLsReplacementPath, + npmLsReplacement, + runCLI, + cliWrapper, + mkTemp +} diff --git a/tests/integration/setup.js b/tests/integration/setup.js index 9f8b57c7..762d1cac 100644 --- a/tests/integration/setup.js +++ b/tests/integration/setup.js @@ -18,30 +18,34 @@ Copyright (c) OWASP Foundation. All Rights Reserved. */ const { spawnSync } = require('child_process') -const path = require('path') +const { join } = require('path') const { getNpmVersion } = require('../_helper') - -const projectRootPath = path.resolve(__dirname, '..', '..') -const demoRootPath = path.resolve(projectRootPath, 'demo'); +const { projectDemoRootPath } = require('./'); (function () { + // skipped for now + return + /* eslint-disable no-unreachable */ + const REQUIRES_INSTALL = [] const npmVersion = getNpmVersion() /* region demos */ + // !! due to inconsistencies between npm6,7,8 - + // some test beds might be skipped if (npmVersion[0] >= 8) { REQUIRES_INSTALL.push( - path.join(demoRootPath, 'alternative-package-registry', 'project'), - path.join(demoRootPath, 'bundled-dependencies', 'project'), - // path.join(demoRootPath, 'deps-from-git', 'project'), - path.join(demoRootPath, 'dev-dependencies', 'project'), - // path.join(demoRootPath, 'juice-shop', 'project'), - path.join(demoRootPath, 'local-dependencies', 'project'), - path.join(demoRootPath, 'local-workspaces', 'project'), - path.join(demoRootPath, 'package-integrity', 'project'), - path.join(demoRootPath, 'package-with-build-id', 'project') + join(projectDemoRootPath, 'alternative-package-registry', 'project'), + join(projectDemoRootPath, 'bundled-dependencies', 'project'), + // join(projectDemoRootPath, 'deps-from-git', 'project'), + join(projectDemoRootPath, 'dev-dependencies', 'project'), + // join(projectDemoRootPath, 'juice-shop', 'project'), + join(projectDemoRootPath, 'local-dependencies', 'project'), + join(projectDemoRootPath, 'local-workspaces', 'project'), + join(projectDemoRootPath, 'package-integrity', 'project'), + join(projectDemoRootPath, 'package-with-build-id', 'project') ) } /* endregion demos */