From 1549a2f33b15f93b30d1aec0a313ac11955def7d Mon Sep 17 00:00:00 2001 From: Sandor Trombitas Date: Wed, 12 Feb 2025 17:13:50 +0200 Subject: [PATCH] chore: extract and refactor cci tests --- .../snyk-code/consistent-ignores.spec.ts | 148 ++++++++++++++++++ .../acceptance/snyk-code/snyk-code.spec.ts | 143 +---------------- 2 files changed, 150 insertions(+), 141 deletions(-) create mode 100644 test/jest/acceptance/snyk-code/consistent-ignores.spec.ts diff --git a/test/jest/acceptance/snyk-code/consistent-ignores.spec.ts b/test/jest/acceptance/snyk-code/consistent-ignores.spec.ts new file mode 100644 index 0000000000..64e0fed87b --- /dev/null +++ b/test/jest/acceptance/snyk-code/consistent-ignores.spec.ts @@ -0,0 +1,148 @@ +import { execSync } from 'child_process'; +import { existsSync, readFileSync, rm, rmdirSync, rmSync, unlinkSync } from 'fs'; +import { resolve } from 'path'; +import { runSnykCLI } from '../../util/runSnykCLI'; + +// This method does some basic checks on the given sarif file +function checkSarif(file: string, expectedIgnoredFindings: number): any { + expect(existsSync(file)).toBe(true); + + const sarifOutput = JSON.parse(readFileSync(file, 'utf8')); + + // Check that the SARIF payload contains all expected fingerprints including identity and snyk/asset/finding/v1 + const fingerprints = sarifOutput.runs[0].results.flatMap( + (result) => result.fingerprints || [], + ); + expect(fingerprints).toContainEqual( + expect.objectContaining({ identity: expect.any(String) }), + ); + expect(fingerprints).toContainEqual( + expect.objectContaining({ + 'snyk/asset/finding/v1': expect.any(String), + }), + ); + + const suppressions = sarifOutput.runs[0].results.filter( + (result) => result.suppressions, + ); + expect(suppressions.length).toBe(expectedIgnoredFindings); + + return sarifOutput; +} + +interface IgnoreTests { + name: string; + expectedExitCode: number; + expectedIgnoredIssuesHigh: number; + expectedIgnoredIssuesMedium: number; + pathToTest: string; +} + +const repoUrl = 'https://github.com/snyk/snyk-goof.git'; +const localPath = '/tmp/snyk-goof'; +const EXIT_CODE_SUCCESS = 0; +const EXIT_CODE_ACTION_NEEDED = 1; + + +const ignoreTestList: IgnoreTests[] = [ + { + name: 'given 4 issues are ignored and 5 open issues are present', + expectedExitCode: EXIT_CODE_ACTION_NEEDED, + expectedIgnoredIssuesHigh: 1, + expectedIgnoredIssuesMedium: 3, + pathToTest: localPath, + }, + { + name: 'given 4 issues are ignored and 0 open issues are present', + expectedExitCode: EXIT_CODE_SUCCESS, + expectedIgnoredIssuesHigh: 1, + expectedIgnoredIssuesMedium: 3, + pathToTest: `${localPath}/routes`, + }, +]; +const projectRoot = resolve(__dirname, '../../../..'); + +describe.each(ignoreTestList)( + `with ignored issues`, + ({ + name, + expectedExitCode, + expectedIgnoredIssuesHigh, + expectedIgnoredIssuesMedium, + pathToTest, + }) => { + const expectedIgnoredIssuesAll = + expectedIgnoredIssuesHigh + expectedIgnoredIssuesMedium; + const sarifFile = `${projectRoot}/sarifOutput.json`; + beforeAll(() => { + if (!existsSync(localPath)) { + // Clone the repository + execSync(`git clone ${repoUrl} ${localPath}`, { stdio: 'inherit' }); + } + }) + afterAll(() => { + if (existsSync(localPath)) { + rmSync(localPath, { recursive: true }) + } + }) + + describe(`${name}`, () => { + jest.setTimeout(2 * 60 * 1000); + afterEach(() => { + // Cleanup SARIF file + try { + unlinkSync(sarifFile); + } catch (error) { + // nothing + } + }); + + it('with --severity-threashold', async () => { + const { stdout, stderr, code } = await runSnykCLI( + `code test ${pathToTest} --severity-threshold=high --sarif-file-output=${sarifFile}`, + { + env: { + INTERNAL_SNYK_CODE_IGNORES_ENABLED: 'true', + ...process.env, + }, + }, + ); + + expect(stderr).toBe(''); + expect(stdout).toContain( + `Ignored issues: ${expectedIgnoredIssuesHigh}`, + ); + expect(stdout.toLowerCase()).not.toContain('[medium]'); + expect(code).toBe(expectedExitCode); + + // Verify SARIF file + const sarifOutput = checkSarif(sarifFile, expectedIgnoredIssuesHigh); + + const levels = sarifOutput.runs[0].results.filter( + (result) => result.level.toLowerCase() == 'warning', + ); + expect(levels.length).toBe(0); + }); + + it('with --include-ignores', async () => { + const { stdout, stderr, code } = await runSnykCLI( + `code test ${pathToTest} --include-ignores --sarif-file-output=${sarifFile}`, + { + env: { + INTERNAL_SNYK_CODE_IGNORES_ENABLED: 'true', + ...process.env, + }, + }, + ); + + expect(stderr).toBe(''); + const ignoredCount = (stdout.match(/\[ IGNORED \]/g) || []).length; + expect(ignoredCount).toBe(expectedIgnoredIssuesAll); + expect(code).toBe(expectedExitCode); + + // Verify SARIF file + checkSarif(sarifFile, expectedIgnoredIssuesAll); + }); + }); + }, +); diff --git a/test/jest/acceptance/snyk-code/snyk-code.spec.ts b/test/jest/acceptance/snyk-code/snyk-code.spec.ts index 8f64205266..b3218f27a4 100644 --- a/test/jest/acceptance/snyk-code/snyk-code.spec.ts +++ b/test/jest/acceptance/snyk-code/snyk-code.spec.ts @@ -5,8 +5,7 @@ import { fakeDeepCodeServer } from '../../../acceptance/deepcode-fake-server'; import { getServerPort } from '../../util/getServerPort'; import { matchers } from 'jest-json-schema'; import { resolve } from 'path'; -import { existsSync, unlinkSync, readFileSync } from 'fs'; -import { execSync } from 'child_process'; +import { existsSync, unlinkSync } from 'fs'; const stripAnsi = require('strip-ansi'); const projectRoot = resolve(__dirname, '../../../..'); @@ -22,47 +21,11 @@ interface Workflow { env: { [key: string]: string | undefined }; } -interface IgnoreTests { - name: string; - expectedExitCode: number; - expectedIgnoredIssuesHigh: number; - expectedIgnoredIssuesMedium: number; - pathToTest: string; -} - const EXIT_CODE_SUCCESS = 0; const EXIT_CODE_ACTION_NEEDED = 1; const EXIT_CODE_FAIL_WITH_ERROR = 2; const EXIT_CODE_NO_SUPPORTED_FILES = 3; -const repoUrl = 'https://github.com/snyk/snyk-goof.git'; -const localPath = '/tmp/snyk-goof'; - -// This method does some basic checks on the given sarif file -function checkSarif(file: string, expectedIgnoredFindings: number): any { - expect(existsSync(file)).toBe(true); - - const sarifOutput = JSON.parse(readFileSync(file, 'utf8')); - - // Check that the SARIF payload contains all expected fingerprints including identity and snyk/asset/finding/v1 - const fingerprints = sarifOutput.runs[0].results.flatMap( - (result) => result.fingerprints || [], - ); - expect(fingerprints).toContainEqual( - expect.objectContaining({ identity: expect.any(String) }), - ); - expect(fingerprints).toContainEqual( - expect.objectContaining({ - 'snyk/asset/finding/v1': expect.any(String), - }), - ); - - const suppressions = sarifOutput.runs[0].results.filter( - (result) => result.suppressions, - ); - expect(suppressions.length).toBe(expectedIgnoredFindings); - return sarifOutput; -} describe('snyk code test', () => { let server: ReturnType; @@ -85,11 +48,6 @@ describe('snyk code test', () => { const emptyProject = resolve(projectRoot, 'test/fixtures/empty'); beforeAll(() => { - if (!existsSync(localPath)) { - // Clone the repository - execSync(`git clone ${repoUrl} ${localPath}`, { stdio: 'inherit' }); - } - return new Promise((resolve, reject) => { try { deepCodeServer = fakeDeepCodeServer(); @@ -200,7 +158,7 @@ describe('snyk code test', () => { it('use remote LCE URL as base when LCE is enabled', async () => { const localCodeEngineUrl = fakeDeepCodeServer(); - localCodeEngineUrl.listen(() => {}); + localCodeEngineUrl.listen(jest.fn); server.setOrgSetting('sast', true); server.setLocalCodeEngineConfiguration({ @@ -574,103 +532,6 @@ describe('snyk code test', () => { console.error('failed to remove file.', error); } }); - - if (type === 'golang/native') { - const ignoreTestList: IgnoreTests[] = [ - { - name: 'given 4 issues are ignored and 5 open issues are present', - expectedExitCode: EXIT_CODE_ACTION_NEEDED, - expectedIgnoredIssuesHigh: 1, - expectedIgnoredIssuesMedium: 3, - pathToTest: localPath, - }, - { - name: 'given 4 issues are ignored and 0 open issues are present', - expectedExitCode: EXIT_CODE_SUCCESS, - expectedIgnoredIssuesHigh: 1, - expectedIgnoredIssuesMedium: 3, - pathToTest: `${localPath}/routes`, - }, - ]; - - const sarifFile = `${projectRoot}/sarifOutput.json`; - - describe.each(ignoreTestList)( - `with ignored issues`, - ({ - name, - expectedExitCode, - expectedIgnoredIssuesHigh, - expectedIgnoredIssuesMedium, - pathToTest, - }) => { - const expectedIgnoredIssuesAll = - expectedIgnoredIssuesHigh + expectedIgnoredIssuesMedium; - - describe(name, () => { - afterEach(() => { - // Cleanup SARIF file - try { - unlinkSync(sarifFile); - } catch (error) { - // nothing - } - }); - - it('with --severity-threashold', async () => { - const { stdout, stderr, code } = await runSnykCLI( - `code test ${pathToTest} --severity-threshold=high --sarif-file-output=${sarifFile}`, - { - env: { - ...process.env, - ...integrationEnv, - }, - }, - ); - - expect(stderr).toBe(''); - expect(stdout).toContain( - `Ignored issues: ${expectedIgnoredIssuesHigh}`, - ); - expect(stdout.toLowerCase()).not.toContain('[medium]'); - expect(code).toBe(expectedExitCode); - - // Verify SARIF file - const sarifOutput = checkSarif( - sarifFile, - expectedIgnoredIssuesHigh, - ); - - const levels = sarifOutput.runs[0].results.filter( - (result) => result.level.toLowerCase() == 'warning', - ); - expect(levels.length).toBe(0); - }); - - it('with --include-ignores', async () => { - const { stdout, stderr, code } = await runSnykCLI( - `code test ${pathToTest} --include-ignores --sarif-file-output=${sarifFile}`, - { - env: { - ...process.env, - ...integrationEnv, - }, - }, - ); - - expect(stderr).toBe(''); - expect( - stdout.toLowerCase().split('[ ignored ]').length - 1, - ).toBe(expectedIgnoredIssuesAll); - expect(code).toBe(expectedExitCode); - - // Verify SARIF file - checkSarif(sarifFile, expectedIgnoredIssuesAll); - }); - }); - }, - ); - } }); }, );