add -z option for getting sizes, but not validating

CMSgov · Feb 14, 2024 · 18bd77a · 18bd77a
1 parent e06e8a0
commit 18bd77a
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 112 deletions.
diff --git a/schemavalidator.cpp b/schemavalidator.cpp
@@ -241,6 +241,7 @@ struct MessageHandler : public BaseReaderHandler<UTF8<>, MessageHandler>
   static list<string> negotiatedTypePath;
   static list<string> inNetworkProviderGroupsPath;
   static list<string> lastUpdatedPath;
+  static list<string> versionPath;
 
   enum State
   {
@@ -266,16 +267,19 @@ struct MessageHandler : public BaseReaderHandler<UTF8<>, MessageHandler>
                            {.fileName = "negotiatedType.json", .path = negotiatedTypePath},
                            {.fileName = "providerGroups.json", .path = inNetworkProviderGroupsPath},
                            {.fileName = "providerReferences.json", .path = providerReferencePath},
-                           {.fileName = "lastUpdated.json", .path = lastUpdatedPath}};
+                           {.fileName = "lastUpdated.json", .path = lastUpdatedPath},
+                           {.fileName = "version.json", .path = versionPath}};
     }
     else if (name == "allowed-amounts")
     {
-      pathsForReporting = {{.fileName = "lastUpdated.json", .path = lastUpdatedPath}};
+      pathsForReporting = {{.fileName = "lastUpdated.json", .path = lastUpdatedPath},
+                           {.fileName = "version.json", .path = versionPath}};
     }
     else if (name == "table-of-contents")
     {
       pathsForReporting = {{.fileName = "allowedAmountFiles.json", .path = tocAllowedAmountPath},
-                           {.fileName = "inNetworkFiles.json", .path = tocInNetworkPath}};
+                           {.fileName = "inNetworkFiles.json", .path = tocInNetworkPath},
+                           {.fileName = "version.json", .path = versionPath}};
     }
     for (auto &ir : pathsForReporting)
     {
@@ -610,6 +614,7 @@ list<string> MessageHandler::additionalInfoPath = {"in_network", "[]", "negotiat
 list<string> MessageHandler::negotiatedTypePath = {"in_network", "[]", "negotiated_rates", "[]", "negotiated_prices", "[]", "negotiated_type"};
 list<string> MessageHandler::inNetworkProviderGroupsPath = {"in_network", "[]", "negotiated_rates", "[]", "provider_groups"};
 list<string> MessageHandler::lastUpdatedPath = {"last_updated_on"};
+list<string> MessageHandler::versionPath = {"version"};
 
 int main(int argc, char *argv[])
 {

diff --git a/src/DockerManager.ts b/src/DockerManager.ts
@@ -1,13 +1,16 @@
-import util, { isArray } from 'util';
+import util from 'util';
 import path from 'path';
 import { exec } from 'child_process';
 import fs from 'fs-extra';
 import temp from 'temp';
 import { logger } from './logger';
+import { bytesToReadableSize } from './utils';
+import { EOL } from 'os';
 
 export class DockerManager {
   containerId = '';
   processedUrls: { uri: string; schema: string; size: number }[] = [];
+  skipRun = false;
 
   constructor(public outputPath = './') {}
 
@@ -37,20 +40,22 @@ export class DockerManager {
         const outputDir = temp.mkdirSync('output');
         const containerOutputPath = path.join(outputDir, 'output.txt');
         const dataSize = fs.statSync(dataPath).size;
-        // copy output files after it finishes
+        this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize });
+        if (this.skipRun) {
+          return { pass: true };
+        }
         const runCommand = this.buildRunCommand(schemaPath, dataPath, outputDir, schemaName);
         logger.info('Running validator container...');
         logger.debug(runCommand);
         return util
           .promisify(exec)(runCommand)
           .then(() => {
-            this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize });
             const containerResult: ContainerResult = { pass: true, locations: {} };
             if (fs.existsSync(containerOutputPath)) {
               if (this.outputPath) {
                 fs.copySync(
                   containerOutputPath,
-                  path.join(this.outputPath, `output${this.processedUrls.length}.txt`)
+                  path.join(this.outputPath, `${this.processedUrls.length}-output.txt`)
                 );
               } else {
                 const outputText = fs.readFileSync(containerOutputPath, 'utf-8');
@@ -61,17 +66,18 @@ export class DockerManager {
             this.moveReports(outputDir, schemaName, containerResult);
             return containerResult;
           })
-          .catch((..._zagwo) => {
-            this.processedUrls.push({ uri: dataUri, schema: schemaName, size: dataSize });
+          .catch(() => {
             const containerResult: ContainerResult = { pass: false, locations: {} };
             if (fs.existsSync(containerOutputPath)) {
+              const outputText = fs.readFileSync(containerOutputPath, 'utf-8');
               if (this.outputPath) {
-                fs.copySync(
-                  containerOutputPath,
-                  path.join(this.outputPath, `output${this.processedUrls.length}.txt`)
+                fs.writeFileSync(
+                  path.join(this.outputPath, `${this.processedUrls.length}-output.txt`),
+                  `${dataUri}${EOL}${bytesToReadableSize(dataSize)}${EOL}${outputText}`
                 );
               } else {
-                const outputText = fs.readFileSync(containerOutputPath, 'utf-8');
+                logger.info(dataUri);
+                logger.info(bytesToReadableSize(dataSize));
                 logger.info(outputText);
               }
             }
@@ -97,7 +103,7 @@ export class DockerManager {
         if (this.outputPath) {
           if (schemaName === 'in-network-rates' && reportFile === 'negotiatedType.json') {
             // convert to map
-            this.convertToPopulationMap(
+            this.writePopulationMap(
               path.join(outputDir, reportFile),
               path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`)
             );
@@ -143,7 +149,7 @@ export class DockerManager {
     });
   }
 
-  private convertToPopulationMap(reportFile: string, outputFile: string) {
+  private writePopulationMap(reportFile: string, outputFile: string) {
     const populationMap = new Map<string, number>();
     const reportContents = fs.readJsonSync(reportFile);
     Object.values(reportContents).forEach((v: any) => {

diff --git a/src/DownloadManager.ts b/src/DownloadManager.ts
@@ -5,7 +5,7 @@ import temp from 'temp';
 import path from 'path';
 import yauzl from 'yauzl';
 import { createGunzip } from 'zlib';
-import { ZipContents, isGzip, isZip } from './utils';
+import { ZipContents, bytesToReadableSize, isGzip, isZip } from './utils';
 import { logger } from './logger';
 
 import { pipeline } from 'stream/promises';
@@ -35,9 +35,7 @@ export class DownloadManager {
           );
         } else if (contentLength > DATA_SIZE_WARNING_THRESHOLD) {
           proceedToDownload = readlineSync.keyInYNStrict(
-            `Data file is ${(contentLength / ONE_MEGABYTE).toFixed(
-              2
-            )} MB in size. Download this file?`
+            `Data file is ${bytesToReadableSize(contentLength)} MB in size. Download this file?`
           );
         } else {
           proceedToDownload = true;

diff --git a/src/SchemaManager.ts b/src/SchemaManager.ts
@@ -85,7 +85,7 @@ export class SchemaManager {
 
   async determineVersion(dataFile: string): Promise<string> {
     return new Promise((resolve, reject) => {
-      logger.debug(`Detecting version for ${dataFile}`);
+      logger.info(`Detecting version for ${dataFile}`);
       const parser = new JSONParser({ paths: ['$.version'], keepStack: false });
       const dataStream = fs.createReadStream(dataFile);
       let foundVersion = '';
@@ -103,7 +103,7 @@ export class SchemaManager {
         });
         parser.on('close', () => {
           if (foundVersion) {
-            logger.debug(`Found version: ${foundVersion}`);
+            logger.info(`Found version: ${foundVersion}`);
             resolve(foundVersion);
           } else {
             reject('No version property available.');
@@ -137,7 +137,7 @@ export class SchemaManager {
                 const versionMatch = lastText.match(versionRegex);
                 if (versionMatch) {
                   const foundVersion = JSON.parse(versionMatch[1]);
-                  logger.debug(`Found version during backwards search: ${foundVersion}`);
+                  logger.info(`Found version during backwards search: ${foundVersion}`);
                   resolve(foundVersion);
                 } else {
                   reject('No version found during backwards search');

diff --git a/src/commands.ts b/src/commands.ts
@@ -71,38 +71,33 @@ export async function validate(dataFile: string, options: OptionValues) {
           dataFile,
           `file://${dataFile}`
         );
-        if (containerResult.pass || true) {
-          if (options.target === 'table-of-contents') {
-            const providerReferences = await assessTocContents(
-              containerResult.locations,
-              schemaManager,
-              dockerManager,
-              downloadManager
-            );
-            await assessReferencedProviders(
-              providerReferences,
-              schemaManager,
-              dockerManager,
-              downloadManager
-            );
-          } else if (
-            options.target === 'in-network-rates' &&
-            containerResult.locations?.providerReference?.length > 0
-          ) {
-            await assessReferencedProviders(
-              containerResult.locations.providerReference,
-              schemaManager,
-              dockerManager,
-              downloadManager
-            );
-          }
-          // make index file
-          writeIndexFile(dockerManager.processedUrls, options.out);
-          // const indexContents = dockerManager.processedUrls
-          //   .map(({ uri, schema }, index) => `${index + 1}\t\t${schema}\t\t${uri}`)
-          //   .join(os.EOL);
-          // fs.writeFileSync(path.join(options.out, 'result-index.txt'), indexContents);
+        dockerManager.skipRun = options.z;
+        if (options.target === 'table-of-contents') {
+          const providerReferences = await assessTocContents(
+            containerResult.locations,
+            schemaManager,
+            dockerManager,
+            downloadManager
+          );
+          await assessReferencedProviders(
+            providerReferences,
+            schemaManager,
+            dockerManager,
+            downloadManager
+          );
+        } else if (
+          options.target === 'in-network-rates' &&
+          containerResult.locations?.providerReference?.length > 0
+        ) {
+          await assessReferencedProviders(
+            containerResult.locations.providerReference,
+            schemaManager,
+            dockerManager,
+            downloadManager
+          );
         }
+        // make index file
+        writeIndexFile(dockerManager.processedUrls, options.out);
       } else {
         logger.error('No schema available - not validating.');
         process.exitCode = 1;
@@ -141,31 +136,30 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) {
               dataFile,
               dataUrl
             );
-            if (containerResult.pass || true) {
-              if (options.target === 'table-of-contents') {
-                const providerReferences = await assessTocContents(
-                  containerResult.locations,
-                  schemaManager,
-                  dockerManager,
-                  downloadManager
-                );
-                await assessReferencedProviders(
-                  providerReferences,
-                  schemaManager,
-                  dockerManager,
-                  downloadManager
-                );
-              } else if (
-                options.target === 'in-network-rates' &&
-                containerResult.locations?.providerReference?.length > 0
-              ) {
-                await assessReferencedProviders(
-                  containerResult.locations.providerReference,
-                  schemaManager,
-                  dockerManager,
-                  downloadManager
-                );
-              }
+            dockerManager.skipRun = options.z;
+            if (options.target === 'table-of-contents') {
+              const providerReferences = await assessTocContents(
+                containerResult.locations,
+                schemaManager,
+                dockerManager,
+                downloadManager
+              );
+              await assessReferencedProviders(
+                providerReferences,
+                schemaManager,
+                dockerManager,
+                downloadManager
+              );
+            } else if (
+              options.target === 'in-network-rates' &&
+              containerResult.locations?.providerReference?.length > 0
+            ) {
+              await assessReferencedProviders(
+                containerResult.locations.providerReference,
+                schemaManager,
+                dockerManager,
+                downloadManager
+              );
             }
             writeIndexFile(dockerManager.processedUrls, options.out);
             return containerResult;
@@ -179,7 +173,7 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) {
                 schemaPath,
                 options.target,
                 dataFile.dataPath,
-                `${dataUrl}:${chosenEntry.fileName}` // TODO see if this is actually useful
+                `${dataUrl}:${chosenEntry.fileName}`
               );
               continuation = readlineSync.keyInYNStrict(
                 'Would you like to validate another file in the ZIP?'

diff --git a/src/index.ts b/src/index.ts
@@ -36,6 +36,7 @@ async function main() {
       'enable strict checking, which prohibits additional properties in data file'
     )
     .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts')
+    .option('-z', 'get file sizes for referenced files, but do not validate them')
     .action(validate);
 
   program
@@ -57,6 +58,7 @@ async function main() {
       'enable strict checking, which prohibits additional properties in data file'
     )
     .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts')
+    .option('-z', 'get file sizes for referenced files, but do not validate them')
     .action((dataUrl, options) => {
       validateFromUrl(dataUrl, options).then(result => {
         if (result) {