diff --git a/schemavalidator.cpp b/schemavalidator.cpp index 3cdea6a..2a40f76 100644 --- a/schemavalidator.cpp +++ b/schemavalidator.cpp @@ -241,6 +241,7 @@ struct MessageHandler : public BaseReaderHandler, MessageHandler> static list negotiatedTypePath; static list inNetworkProviderGroupsPath; static list lastUpdatedPath; + static list versionPath; enum State { @@ -266,16 +267,19 @@ struct MessageHandler : public BaseReaderHandler, MessageHandler> {.fileName = "negotiatedType.json", .path = negotiatedTypePath}, {.fileName = "providerGroups.json", .path = inNetworkProviderGroupsPath}, {.fileName = "providerReferences.json", .path = providerReferencePath}, - {.fileName = "lastUpdated.json", .path = lastUpdatedPath}}; + {.fileName = "lastUpdated.json", .path = lastUpdatedPath}, + {.fileName = "version.json", .path = versionPath}}; } else if (name == "allowed-amounts") { - pathsForReporting = {{.fileName = "lastUpdated.json", .path = lastUpdatedPath}}; + pathsForReporting = {{.fileName = "lastUpdated.json", .path = lastUpdatedPath}, + {.fileName = "version.json", .path = versionPath}}; } else if (name == "table-of-contents") { pathsForReporting = {{.fileName = "allowedAmountFiles.json", .path = tocAllowedAmountPath}, - {.fileName = "inNetworkFiles.json", .path = tocInNetworkPath}}; + {.fileName = "inNetworkFiles.json", .path = tocInNetworkPath}, + {.fileName = "version.json", .path = versionPath}}; } for (auto &ir : pathsForReporting) { @@ -610,6 +614,7 @@ list MessageHandler::additionalInfoPath = {"in_network", "[]", "negotiat list MessageHandler::negotiatedTypePath = {"in_network", "[]", "negotiated_rates", "[]", "negotiated_prices", "[]", "negotiated_type"}; list MessageHandler::inNetworkProviderGroupsPath = {"in_network", "[]", "negotiated_rates", "[]", "provider_groups"}; list MessageHandler::lastUpdatedPath = {"last_updated_on"}; +list MessageHandler::versionPath = {"version"}; int main(int argc, char *argv[]) { diff --git a/src/DockerManager.ts b/src/DockerManager.ts index ffd0453..60e4cd3 100644 --- a/src/DockerManager.ts +++ b/src/DockerManager.ts @@ -1,13 +1,16 @@ -import util, { isArray } from 'util'; +import util from 'util'; import path from 'path'; import { exec } from 'child_process'; import fs from 'fs-extra'; import temp from 'temp'; import { logger } from './logger'; +import { bytesToReadableSize } from './utils'; +import { EOL } from 'os'; export class DockerManager { containerId = ''; processedUrls: { uri: string; schema: string; size: number }[] = []; + skipRun = false; constructor(public outputPath = './') {} @@ -37,20 +40,22 @@ export class DockerManager { const outputDir = temp.mkdirSync('output'); const containerOutputPath = path.join(outputDir, 'output.txt'); const dataSize = fs.statSync(dataPath).size; - // copy output files after it finishes + this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize }); + if (this.skipRun) { + return { pass: true }; + } const runCommand = this.buildRunCommand(schemaPath, dataPath, outputDir, schemaName); logger.info('Running validator container...'); logger.debug(runCommand); return util .promisify(exec)(runCommand) .then(() => { - this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize }); const containerResult: ContainerResult = { pass: true, locations: {} }; if (fs.existsSync(containerOutputPath)) { if (this.outputPath) { fs.copySync( containerOutputPath, - path.join(this.outputPath, `output${this.processedUrls.length}.txt`) + path.join(this.outputPath, `${this.processedUrls.length}-output.txt`) ); } else { const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); @@ -61,17 +66,18 @@ export class DockerManager { this.moveReports(outputDir, schemaName, containerResult); return containerResult; }) - .catch((..._zagwo) => { - this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize }); + .catch(() => { const containerResult: ContainerResult = { pass: false, locations: {} }; if (fs.existsSync(containerOutputPath)) { + const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); if (this.outputPath) { - fs.copySync( - containerOutputPath, - path.join(this.outputPath, `output${this.processedUrls.length}.txt`) + fs.writeFileSync( + path.join(this.outputPath, `${this.processedUrls.length}-output.txt`), + `${dataUri}${EOL}${bytesToReadableSize(dataSize)}${EOL}${outputText}` ); } else { - const outputText = fs.readFileSync(containerOutputPath, 'utf-8'); + logger.info(dataUri); + logger.info(bytesToReadableSize(dataSize)); logger.info(outputText); } } @@ -97,7 +103,7 @@ export class DockerManager { if (this.outputPath) { if (schemaName === 'in-network-rates' && reportFile === 'negotiatedType.json') { // convert to map - this.convertToPopulationMap( + this.writePopulationMap( path.join(outputDir, reportFile), path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`) ); @@ -143,7 +149,7 @@ export class DockerManager { }); } - private convertToPopulationMap(reportFile: string, outputFile: string) { + private writePopulationMap(reportFile: string, outputFile: string) { const populationMap = new Map(); const reportContents = fs.readJsonSync(reportFile); Object.values(reportContents).forEach((v: any) => { diff --git a/src/DownloadManager.ts b/src/DownloadManager.ts index dd4c4e5..da52ef5 100644 --- a/src/DownloadManager.ts +++ b/src/DownloadManager.ts @@ -5,7 +5,7 @@ import temp from 'temp'; import path from 'path'; import yauzl from 'yauzl'; import { createGunzip } from 'zlib'; -import { ZipContents, isGzip, isZip } from './utils'; +import { ZipContents, bytesToReadableSize, isGzip, isZip } from './utils'; import { logger } from './logger'; import { pipeline } from 'stream/promises'; @@ -35,9 +35,7 @@ export class DownloadManager { ); } else if (contentLength > DATA_SIZE_WARNING_THRESHOLD) { proceedToDownload = readlineSync.keyInYNStrict( - `Data file is ${(contentLength / ONE_MEGABYTE).toFixed( - 2 - )} MB in size. Download this file?` + `Data file is ${bytesToReadableSize(contentLength)} MB in size. Download this file?` ); } else { proceedToDownload = true; diff --git a/src/SchemaManager.ts b/src/SchemaManager.ts index a08dcca..c4a803b 100644 --- a/src/SchemaManager.ts +++ b/src/SchemaManager.ts @@ -85,7 +85,7 @@ export class SchemaManager { async determineVersion(dataFile: string): Promise { return new Promise((resolve, reject) => { - logger.debug(`Detecting version for ${dataFile}`); + logger.info(`Detecting version for ${dataFile}`); const parser = new JSONParser({ paths: ['$.version'], keepStack: false }); const dataStream = fs.createReadStream(dataFile); let foundVersion = ''; @@ -103,7 +103,7 @@ export class SchemaManager { }); parser.on('close', () => { if (foundVersion) { - logger.debug(`Found version: ${foundVersion}`); + logger.info(`Found version: ${foundVersion}`); resolve(foundVersion); } else { reject('No version property available.'); @@ -137,7 +137,7 @@ export class SchemaManager { const versionMatch = lastText.match(versionRegex); if (versionMatch) { const foundVersion = JSON.parse(versionMatch[1]); - logger.debug(`Found version during backwards search: ${foundVersion}`); + logger.info(`Found version during backwards search: ${foundVersion}`); resolve(foundVersion); } else { reject('No version found during backwards search'); diff --git a/src/commands.ts b/src/commands.ts index 012bf30..30b7ada 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -71,38 +71,33 @@ export async function validate(dataFile: string, options: OptionValues) { dataFile, `file://${dataFile}` ); - if (containerResult.pass || true) { - if (options.target === 'table-of-contents') { - const providerReferences = await assessTocContents( - containerResult.locations, - schemaManager, - dockerManager, - downloadManager - ); - await assessReferencedProviders( - providerReferences, - schemaManager, - dockerManager, - downloadManager - ); - } else if ( - options.target === 'in-network-rates' && - containerResult.locations?.providerReference?.length > 0 - ) { - await assessReferencedProviders( - containerResult.locations.providerReference, - schemaManager, - dockerManager, - downloadManager - ); - } - // make index file - writeIndexFile(dockerManager.processedUrls, options.out); - // const indexContents = dockerManager.processedUrls - // .map(({ uri, schema }, index) => `${index + 1}\t\t${schema}\t\t${uri}`) - // .join(os.EOL); - // fs.writeFileSync(path.join(options.out, 'result-index.txt'), indexContents); + dockerManager.skipRun = options.z; + if (options.target === 'table-of-contents') { + const providerReferences = await assessTocContents( + containerResult.locations, + schemaManager, + dockerManager, + downloadManager + ); + await assessReferencedProviders( + providerReferences, + schemaManager, + dockerManager, + downloadManager + ); + } else if ( + options.target === 'in-network-rates' && + containerResult.locations?.providerReference?.length > 0 + ) { + await assessReferencedProviders( + containerResult.locations.providerReference, + schemaManager, + dockerManager, + downloadManager + ); } + // make index file + writeIndexFile(dockerManager.processedUrls, options.out); } else { logger.error('No schema available - not validating.'); process.exitCode = 1; @@ -141,31 +136,30 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { dataFile, dataUrl ); - if (containerResult.pass || true) { - if (options.target === 'table-of-contents') { - const providerReferences = await assessTocContents( - containerResult.locations, - schemaManager, - dockerManager, - downloadManager - ); - await assessReferencedProviders( - providerReferences, - schemaManager, - dockerManager, - downloadManager - ); - } else if ( - options.target === 'in-network-rates' && - containerResult.locations?.providerReference?.length > 0 - ) { - await assessReferencedProviders( - containerResult.locations.providerReference, - schemaManager, - dockerManager, - downloadManager - ); - } + dockerManager.skipRun = options.z; + if (options.target === 'table-of-contents') { + const providerReferences = await assessTocContents( + containerResult.locations, + schemaManager, + dockerManager, + downloadManager + ); + await assessReferencedProviders( + providerReferences, + schemaManager, + dockerManager, + downloadManager + ); + } else if ( + options.target === 'in-network-rates' && + containerResult.locations?.providerReference?.length > 0 + ) { + await assessReferencedProviders( + containerResult.locations.providerReference, + schemaManager, + dockerManager, + downloadManager + ); } writeIndexFile(dockerManager.processedUrls, options.out); return containerResult; @@ -179,7 +173,7 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { schemaPath, options.target, dataFile.dataPath, - `${dataUrl}:${chosenEntry.fileName}` // TODO see if this is actually useful + `${dataUrl}:${chosenEntry.fileName}` ); continuation = readlineSync.keyInYNStrict( 'Would you like to validate another file in the ZIP?' diff --git a/src/index.ts b/src/index.ts index f715bb8..086faf6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ async function main() { 'enable strict checking, which prohibits additional properties in data file' ) .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') + .option('-z', 'get file sizes for referenced files, but do not validate them') .action(validate); program @@ -57,6 +58,7 @@ async function main() { 'enable strict checking, which prohibits additional properties in data file' ) .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') + .option('-z', 'get file sizes for referenced files, but do not validate them') .action((dataUrl, options) => { validateFromUrl(dataUrl, options).then(result => { if (result) { diff --git a/src/utils.ts b/src/utils.ts index 3e2c697..e338a40 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -11,6 +11,7 @@ import { DownloadManager } from './DownloadManager'; const ONE_KILOBYTE = 1024; const ONE_MEGABYTE = 1024 * ONE_KILOBYTE; +const ONE_GIGABYTE = 1024 * ONE_MEGABYTE; export type ZipContents = { zipFile: yauzl.ZipFile; @@ -194,10 +195,6 @@ export async function validateTocContents( downloadManager: DownloadManager ): Promise { temp.track(); - let tempOutput = ''; - if (dockerManager.outputPath?.length > 0) { - tempOutput = path.join(temp.mkdirSync('contents'), 'contained-result'); - } let providerReferences: Set; if (inNetwork.length > 0) { if (schemaManager.shouldDetectVersion) { @@ -205,16 +202,14 @@ export async function validateTocContents( inNetwork, schemaManager, dockerManager, - downloadManager, - tempOutput + downloadManager ); } else { providerReferences = await validateInNetworkFixedVersion( inNetwork, schemaManager, dockerManager, - downloadManager, - tempOutput + downloadManager ); } } @@ -224,16 +219,14 @@ export async function validateTocContents( allowedAmount, schemaManager, dockerManager, - downloadManager, - tempOutput + downloadManager ); } else { await validateAllowedAmountsFixedVersion( allowedAmount, schemaManager, dockerManager, - downloadManager, - tempOutput + downloadManager ); } } @@ -244,8 +237,7 @@ async function validateInNetworkFixedVersion( inNetwork: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - downloadManager: DownloadManager, - tempOutput: string + downloadManager: DownloadManager ) { const providerReferences: Set = new Set(); await schemaManager.useSchema('in-network-rates').then(async schemaPath => { @@ -302,8 +294,7 @@ async function validateInNetworkDetectedVersion( inNetwork: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - downloadManager: DownloadManager, - tempOutput: string + downloadManager: DownloadManager ) { const providerReferences: Set = new Set(); for (const dataUrl of inNetwork) { @@ -356,8 +347,7 @@ async function validateAllowedAmountsFixedVersion( allowedAmount: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - downloadManager: DownloadManager, - tempOutput: string + downloadManager: DownloadManager ) { await schemaManager.useSchema('allowed-amounts').then(async schemaPath => { if (schemaPath != null) { @@ -402,8 +392,7 @@ async function validateAllowedAmountsDetectedVersion( allowedAmount: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - downloadManager: DownloadManager, - tempOutput: string + downloadManager: DownloadManager ) { for (const dataUrl of allowedAmount) { if (dockerManager.processedUrls.some(existing => existing.uri === dataUrl)) { @@ -471,10 +460,6 @@ export async function validateReferencedProviders( downloadManager: DownloadManager ) { temp.track(); - let tempOutput = ''; - if (dockerManager.outputPath?.length > 0) { - tempOutput = path.join(temp.mkdirSync('providers'), 'contained-result'); - } if (providerReferences.length > 0) { await schemaManager.useSchema('provider-reference').then(async schemaPath => { if (schemaPath != null) { @@ -513,21 +498,14 @@ export function writeIndexFile(urls: DockerManager['processedUrls'], outputDir: const schemaSizeTotals = new Map(); const indexContents = urls .map((record, index) => { - let size: string; - if (record.size > ONE_MEGABYTE) { - size = `${record.size / ONE_MEGABYTE}MB`; - } else if (record.size > ONE_KILOBYTE) { - size = `${record.size / ONE_KILOBYTE}KB`; - } else { - size = `${record.size}B`; - } + const size = bytesToReadableSize(record.size); schemaSizeTotals.set(record.schema, (schemaSizeTotals.get(record.schema) ?? 0) + record.size); return `${index + 1}\t${record.schema}\t${record.uri}\t${size}`; }) .join(os.EOL); const sizeInfo = [...schemaSizeTotals.entries()] .map(([schema, size]) => { - return `${schema}: ${size / ONE_MEGABYTE}MB`; + return `${schema}: ${bytesToReadableSize(size)}`; }) .join(os.EOL); fs.writeFileSync( @@ -535,3 +513,15 @@ export function writeIndexFile(urls: DockerManager['processedUrls'], outputDir: `${indexContents}${os.EOL}${os.EOL}${sizeInfo}` ); } + +export function bytesToReadableSize(bytes: number): string { + if (bytes > ONE_GIGABYTE) { + return `${(bytes / ONE_GIGABYTE).toFixed(2)} GB`; + } else if (bytes > ONE_MEGABYTE) { + return `${(bytes / ONE_MEGABYTE).toFixed(2)} MB`; + } else if (bytes > ONE_KILOBYTE) { + return `${bytes / ONE_KILOBYTE} KB`; + } else { + return `${bytes} B`; + } +}