diff --git a/package-lock.json b/package-lock.json index 336e767..a9fe3ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cms-mrf-validator", - "version": "2.0.0", + "version": "2.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "cms-mrf-validator", - "version": "2.0.0", + "version": "2.1.0", "license": "Apache-2.0", "dependencies": { "@streamparser/json": "^0.0.17", diff --git a/src/DockerManager.ts b/src/DockerManager.ts index fcbb175..0625dac 100644 --- a/src/DockerManager.ts +++ b/src/DockerManager.ts @@ -7,8 +7,9 @@ import { logger } from './logger'; export class DockerManager { containerId = ''; + processedUrls: { uri: string; schema: string }[] = []; - constructor(public outputPath = '') {} + constructor(public outputPath = './') {} private async initContainerId(): Promise { this.containerId = await util @@ -24,7 +25,7 @@ export class DockerManager { schemaPath: string, schemaName: string, dataPath: string, - outputPath = this.outputPath + dataUri: string ): Promise { try { if (this.containerId.length === 0) { @@ -35,7 +36,7 @@ export class DockerManager { temp.track(); const outputDir = temp.mkdirSync('output'); const containerOutputPath = path.join(outputDir, 'output.txt'); - const containerLocationPath = path.join(outputDir, 'locations.json'); + const containerReportsPath = path.join(outputDir, 'reports.json'); // copy output files after it finishes const runCommand = this.buildRunCommand(schemaPath, dataPath, outputDir, schemaName); logger.info('Running validator container...'); @@ -43,28 +44,67 @@ export class DockerManager { return util .promisify(exec)(runCommand) .then(() => { + this.processedUrls.push({ uri: dataUri, schema: schemaName }); const containerResult: ContainerResult = { pass: true }; if (fs.existsSync(containerOutputPath)) { - if (outputPath) { - fs.copySync(containerOutputPath, outputPath); + if (this.outputPath) { + fs.copySync( + containerOutputPath, + path.join(this.outputPath, `output${this.processedUrls.length}.txt`) + ); } else { const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); logger.info(outputText); } } - if (fs.existsSync(containerLocationPath)) { + if (fs.existsSync(containerReportsPath)) { + if (this.outputPath) { + fs.copySync( + containerReportsPath, + path.join(this.outputPath, `reports${this.processedUrls.length}.json`) + ); + } try { - containerResult.locations = fs.readJsonSync(containerLocationPath); + if (schemaName === 'table-of-contents') { + // if key ends with .allowed_amount_file: it's an object, grab .location + // if key ends with .in_network_files: it's an array, foreach grab .location + const reports = fs.readJsonSync(containerReportsPath); + containerResult.locations = { allowedAmount: [], inNetwork: [] }; + Object.keys(reports).forEach((key: string) => { + if (key.endsWith('.allowed_amount_file') && reports[key].location != null) { + containerResult.locations.allowedAmount.push(reports[key].location); + } else if (key.endsWith('.in_network_files')) { + reports[key]?.forEach((inNetwork: any) => { + if (inNetwork?.location != null) { + containerResult.locations.inNetwork.push(inNetwork.location); + } + }); + } + }); + } else if (schemaName === 'in-network-rates') { + // if key ends with .location: it's a string, grab it + const reports = fs.readJsonSync(containerReportsPath); + containerResult.locations = { providerReference: [] }; + Object.keys(reports).forEach((key: string) => { + if (key.endsWith('.location')) { + containerResult.locations.providerReference.push(reports[key]); + } + }); + } } catch (err) { - // something went wrong when reading the location file that the validator produced + // don't know either } } return containerResult; }) - .catch(() => { + .catch((..._zagwo) => { + this.processedUrls.push({ uri: dataUri, schema: schemaName }); if (fs.existsSync(containerOutputPath)) { - if (outputPath) { - fs.copySync(containerOutputPath, outputPath); + if (this.outputPath) { + fs.copySync( + containerOutputPath, + path.join(this.outputPath, `output${this.processedUrls.length}.txt`) + ); } else { const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); logger.info(outputText); @@ -103,7 +143,7 @@ export class DockerManager { outputDir )}":/output/ ${ this.containerId - } "schema/${schemaFile}" "data/${dataFile}" -o "output/" -s ${schemaName}`; + } "schema/${schemaFile}" "data/${dataFile}" "output/" -s ${schemaName}`; } } diff --git a/src/commands.ts b/src/commands.ts index c641419..f127db2 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -1,6 +1,7 @@ import util from 'util'; import path from 'path'; import { exec } from 'child_process'; +import os from 'os'; import fs from 'fs-extra'; import readlineSync from 'readline-sync'; import { OptionValues } from 'commander'; @@ -25,6 +26,7 @@ export async function validate(dataFile: string, options: OptionValues) { process.exitCode = 1; return; } + fs.ensureDirSync(options.out); const schemaManager = new SchemaManager(); await schemaManager.ensureRepo(); schemaManager.strict = options.strict; @@ -65,9 +67,10 @@ export async function validate(dataFile: string, options: OptionValues) { const containerResult = await dockerManager.runContainer( schemaPath, options.target, - dataFile + dataFile, + `file://${dataFile}` ); - if (containerResult.pass) { + if (containerResult.pass || true) { if (options.target === 'table-of-contents') { const providerReferences = await assessTocContents( containerResult.locations, @@ -92,6 +95,11 @@ export async function validate(dataFile: string, options: OptionValues) { downloadManager ); } + // make index file + const indexContents = dockerManager.processedUrls + .map(({ uri, schema }, index) => `${index + 1}\t\t${schema}\t\t${uri}`) + .join(os.EOL); + fs.writeFileSync(path.join(options.out, 'result-index.txt'), indexContents); } } else { logger.error('No schema available - not validating.'); @@ -107,6 +115,7 @@ export async function validate(dataFile: string, options: OptionValues) { export async function validateFromUrl(dataUrl: string, options: OptionValues) { temp.track(); + fs.ensureDirSync(options.out); const downloadManager = new DownloadManager(options.yesAll); if (await downloadManager.checkDataUrl(dataUrl)) { const schemaManager = new SchemaManager(); @@ -127,9 +136,10 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { const containerResult = await dockerManager.runContainer( schemaPath, options.target, - dataFile + dataFile, + dataUrl ); - if (containerResult.pass) { + if (containerResult.pass || true) { if (options.target === 'table-of-contents') { const providerReferences = await assessTocContents( containerResult.locations, @@ -162,13 +172,23 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { while (continuation === true) { const chosenEntry = chooseJsonFile(dataFile.jsonEntries); await getEntryFromZip(dataFile.zipFile, chosenEntry, dataFile.dataPath); - await dockerManager.runContainer(schemaPath, options.target, dataFile.dataPath); + await dockerManager.runContainer( + schemaPath, + options.target, + dataFile.dataPath, + `${dataUrl}:${chosenEntry.fileName}` // TODO see if this is actually useful + ); continuation = readlineSync.keyInYNStrict( 'Would you like to validate another file in the ZIP?' ); } dataFile.zipFile.close(); } + // make index file + const indexContents = dockerManager.processedUrls + .map(({ uri, schema }, index) => `${index + 1}\t\t${schema}\t\t${uri}`) + .join(os.EOL); + fs.writeFileSync(path.join(options.out, 'result-index.txt'), indexContents); } else { logger.error('No schema available - not validating.'); process.exitCode = 1; diff --git a/src/index.ts b/src/index.ts index a1493a5..f715bb8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,7 +25,7 @@ async function main() { .usage(' [options]') .argument('', 'path to data file to validate') .option('--schema-version ', 'version of schema to use for validation') - .option('-o, --out ', 'output path') + .option('-o, --out ', 'output directory', './') .addOption( new Option('-t, --target ', 'name of schema to use') .choices(config.AVAILABLE_SCHEMAS) @@ -46,7 +46,7 @@ async function main() { .usage(' [options]') .argument('', 'URL to data file to validate') .option('--schema-version ', 'version of schema to use for validation') - .option('-o, --out ', 'output path') + .option('-o, --out ', 'output directory', './') .addOption( new Option('-t, --target ', 'name of schema to use') .choices(config.AVAILABLE_SCHEMAS) diff --git a/src/utils.ts b/src/utils.ts index fe00177..2914f3c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,7 +3,6 @@ import path from 'path'; import fs from 'fs-extra'; import temp from 'temp'; import yauzl from 'yauzl'; -import { EOL } from 'os'; import { DockerManager } from './DockerManager'; import { SchemaManager } from './SchemaManager'; import { logger } from './logger'; @@ -69,9 +68,7 @@ export function isGzip(contentType: string, url: string): boolean { } export function isZip(contentType: string, url: string): boolean { - return ( - contentType === 'application/zip' || /\.zip(\?|$)/.test(url) - ); + return contentType === 'application/zip' || /\.zip(\?|$)/.test(url); } export function chooseJsonFile(entries: yauzl.Entry[]): yauzl.Entry { @@ -250,6 +247,10 @@ async function validateInNetworkFixedVersion( await schemaManager.useSchema('in-network-rates').then(async schemaPath => { if (schemaPath != null) { for (const dataUrl of inNetwork) { + if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { + logger.info(`File ${dataUrl} already processed, skipping...`); + continue; + } try { if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); @@ -271,7 +272,7 @@ async function validateInNetworkFixedVersion( schemaPath, 'in-network-rates', dataPath, - tempOutput + dataUrl ); if ( containedResult.pass && @@ -281,13 +282,6 @@ async function validateInNetworkFixedVersion( providerReferences.add(prf) ); } - if (tempOutput.length > 0) { - appendResults( - tempOutput, - dockerManager.outputPath, - `${dataUrl} - in-network${EOL}` - ); - } } } else { logger.error(`Could not download file: ${dataUrl}`); @@ -312,6 +306,10 @@ async function validateInNetworkDetectedVersion( ) { const providerReferences: Set = new Set(); for (const dataUrl of inNetwork) { + if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { + logger.info(`File ${dataUrl} already processed, skipping...`); + continue; + } try { if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); @@ -333,7 +331,7 @@ async function validateInNetworkDetectedVersion( schemaPath, 'in-network-rates', dataPath, - tempOutput + dataUrl ); } }) @@ -346,13 +344,6 @@ async function validateInNetworkDetectedVersion( providerReferences.add(prf) ); } - if (tempOutput.length > 0) { - appendResults( - tempOutput, - dockerManager.outputPath, - `${dataUrl} - in-network${EOL}` - ); - } }); } } @@ -373,6 +364,10 @@ async function validateAllowedAmountsFixedVersion( await schemaManager.useSchema('allowed-amounts').then(async schemaPath => { if (schemaPath != null) { for (const dataUrl of allowedAmount) { + if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { + logger.info(`File ${dataUrl} already processed, skipping...`); + continue; + } try { if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); @@ -390,14 +385,7 @@ async function validateAllowedAmountsFixedVersion( } }) .catch(() => {}); - await dockerManager.runContainer(schemaPath, 'allowed-amounts', dataPath, tempOutput); - if (tempOutput.length > 0) { - appendResults( - tempOutput, - dockerManager.outputPath, - `${dataUrl} - allowed-amounts${EOL}` - ); - } + await dockerManager.runContainer(schemaPath, 'allowed-amounts', dataPath, dataUrl); } } else { logger.error(`Could not download file: ${dataUrl}`); @@ -420,6 +408,10 @@ async function validateAllowedAmountsDetectedVersion( tempOutput: string ) { for (const dataUrl of allowedAmount) { + if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { + logger.info(`File ${dataUrl} already processed, skipping...`); + continue; + } try { if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); @@ -437,23 +429,10 @@ async function validateAllowedAmountsDetectedVersion( }) .then(schemaPath => { if (schemaPath != null) { - return dockerManager.runContainer( - schemaPath, - 'allowed-amounts', - dataPath, - tempOutput - ); + return dockerManager.runContainer(schemaPath, 'allowed-amounts', dataPath, dataUrl); } }) - .then(_containedResult => { - if (tempOutput.length > 0) { - appendResults( - tempOutput, - dockerManager.outputPath, - `${dataUrl} - allowed-amounts${EOL}` - ); - } - }); + .then(_containedResult => {}); } } } catch (err) { @@ -470,7 +449,7 @@ export async function assessReferencedProviders( ) { if (providerReferences.length > 0) { const fileText = providerReferences.length === 1 ? 'this file' : 'these files'; - if (providerReferences.length === 1) { + if (providerReferences.length > 1) { logger.info(`In-network file(s) refer to ${fileText}:`); logger.info('== Provider Reference =='); providerReferences.forEach(prf => logger.info(`* ${prf}`)); @@ -504,6 +483,10 @@ export async function validateReferencedProviders( schemaManager.useSchema('provider-reference').then(async schemaPath => { if (schemaPath != null) { for (const dataUrl of providerReferences) { + if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { + logger.info(`File ${dataUrl} already processed, skipping...`); + continue; + } try { if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); @@ -513,15 +496,8 @@ export async function validateReferencedProviders( schemaPath, 'provider-reference', dataPath, - tempOutput + dataUrl ); - if (tempOutput.length > 0) { - appendResults( - tempOutput, - dockerManager.outputPath, - `${dataUrl} - provider-reference${EOL}` - ); - } } } else { logger.error(`Could not download file: ${dataUrl}`);