Skip to content

Commit

Permalink
safety checks around big files
Browse files Browse the repository at this point in the history
  • Loading branch information
@mint-thompson
mint-thompson committed Mar 6, 2024
1 parent 3913d0a commit 9e087fb
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 86 deletions.
143 changes: 83 additions & 60 deletions src/DockerManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export class DockerManager {
return util
.promisify(exec)(runCommand)
.then(() => {
logger.debug('validator done.');
const containerResult: ContainerResult = { pass: true, locations: {} };
if (fs.existsSync(containerOutputPath)) {
if (this.outputPath) {
Expand All @@ -68,20 +69,32 @@ export class DockerManager {
return containerResult;
})
.catch(() => {
logger.debug('validator done.');
const containerResult: ContainerResult = { pass: false, locations: {} };
if (fs.existsSync(containerOutputPath)) {
const outputText = fs.readFileSync(containerOutputPath, 'utf-8');
if (this.outputPath) {
fs.writeFileSync(
path.join(this.outputPath, `${this.processedUrls.length}-output.txt`),
`${dataUri}${EOL}${bytesToReadableSize(dataSize)}${EOL}${outputText}`
);
} else {
logger.info(dataUri);
logger.info(bytesToReadableSize(dataSize));
logger.info(outputText);
try {
if (fs.existsSync(containerOutputPath)) {
const outputText = fs.readFileSync(containerOutputPath, 'utf-8');
if (this.outputPath) {
fs.writeFileSync(
path.join(this.outputPath, `${this.processedUrls.length}-output.txt`),
`${dataUri}${EOL}${bytesToReadableSize(dataSize)}${EOL}${outputText}`
);
} else {
logger.info(dataUri);
logger.info(bytesToReadableSize(dataSize));
logger.info(outputText);
}
}
} catch (err) {
logger.error(err);
logger.info(dataUri);
logger.info(bytesToReadableSize(dataSize));
fs.copyFileSync(
containerOutputPath,
path.join(this.outputPath, `${this.processedUrls.length}-output.txt`)
);
}

this.moveReports(outputDir, schemaName, containerResult);
process.exitCode = 1;
return containerResult;
Expand All @@ -100,65 +113,75 @@ export class DockerManager {

private moveReports(outputDir: string, schemaName: string, containerResult: ContainerResult) {
fs.readdirSync(outputDir).forEach(reportFile => {
if (reportFile.endsWith('.json') && reportFile != 'errors.json') {
if (this.outputPath) {
if (schemaName === 'in-network-rates' && reportFile === 'negotiatedType.json') {
// convert to map
this.writePopulationMap(
path.join(outputDir, reportFile),
path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`)
);
} else {
fs.copySync(
path.join(outputDir, reportFile),
path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`)
);
try {
logger.debug(`moving report ${reportFile}`);
if (reportFile.endsWith('.json') && reportFile != 'errors.json') {
if (this.outputPath) {
if (schemaName === 'in-network-rates' && reportFile === 'negotiatedType.json') {
// convert to map
this.writePopulationMap(
path.join(outputDir, reportFile),
path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`)
);
} else {
fs.copySync(
path.join(outputDir, reportFile),
path.join(this.outputPath, `${this.processedUrls.length}-${reportFile}`)
);
}
}
}
}
if (schemaName === 'table-of-contents') {
if (reportFile === 'allowedAmountFiles.json') {
const allowedAmountFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.allowedAmount = [];
Object.keys(allowedAmountFiles).forEach((key: string) => {
if (typeof allowedAmountFiles[key].location === 'string') {
containerResult.locations.allowedAmount.push(allowedAmountFiles[key].location);
}
});
} else if (reportFile === 'inNetworkFiles.json') {
const inNetworkFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.inNetwork = [];
Object.keys(inNetworkFiles).forEach((key: string) => {
if (Array.isArray(inNetworkFiles[key])) {
inNetworkFiles[key].forEach((entry: any) => {
if (typeof entry.location === 'string') {
containerResult.locations.inNetwork.push(entry.location);
}
});
if (schemaName === 'table-of-contents') {
if (reportFile === 'allowedAmountFiles.json') {
const allowedAmountFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.allowedAmount = [];
Object.keys(allowedAmountFiles).forEach((key: string) => {
if (typeof allowedAmountFiles[key].location === 'string') {
containerResult.locations.allowedAmount.push(allowedAmountFiles[key].location);
}
});
} else if (reportFile === 'inNetworkFiles.json') {
const inNetworkFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.inNetwork = [];
Object.keys(inNetworkFiles).forEach((key: string) => {
if (Array.isArray(inNetworkFiles[key])) {
inNetworkFiles[key].forEach((entry: any) => {
if (typeof entry.location === 'string') {
containerResult.locations.inNetwork.push(entry.location);
}
});
}
});
}
} else if (schemaName === 'in-network-rates' && reportFile === 'providerReferences.json') {
const providerReferenceFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.providerReference = [];
Object.keys(providerReferenceFiles).forEach((key: string) => {
if (typeof providerReferenceFiles[key] === 'string') {
containerResult.locations.providerReference.push(providerReferenceFiles[key]);
}
});
}
} else if (schemaName === 'in-network-rates' && reportFile === 'providerReferences.json') {
const providerReferenceFiles = fs.readJsonSync(path.join(outputDir, reportFile));
containerResult.locations.providerReference = [];
Object.keys(providerReferenceFiles).forEach((key: string) => {
if (typeof providerReferenceFiles[key] === 'string') {
containerResult.locations.providerReference.push(providerReferenceFiles[key]);
}
});
} catch (err) {
logger.error(`problem moving report ${reportFile}: ${err}`);
}
});
}

private writePopulationMap(reportFile: string, outputFile: string) {
const populationMap = new Map<string, number>();
const reportContents = fs.readJsonSync(reportFile);
Object.values(reportContents).forEach((v: any) => {
if (typeof v === 'string') {
populationMap.set(v, (populationMap.get(v) ?? 0) + 1);
}
});
fs.writeJsonSync(outputFile, Object.fromEntries(populationMap.entries()));
try {
const populationMap = new Map<string, number>();
const reportContents = fs.readJsonSync(reportFile);
Object.values(reportContents).forEach((v: any) => {
if (typeof v === 'string') {
populationMap.set(v, (populationMap.get(v) ?? 0) + 1);
}
});
fs.writeJsonSync(outputFile, Object.fromEntries(populationMap.entries()));
} catch (err) {
logger.error(`population map problem: ${err}`);
fs.copyFileSync(reportFile, outputFile);
}
}

buildRunCommand(
Expand Down
47 changes: 21 additions & 26 deletions src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,6 @@ function showMenuOptions(currentPage: number, maxPage: number, items: string[])
logger.menu(commandsToShow.join(' | '));
}

export function appendResults(source: string, destination: string, prefixData: string = '') {
try {
const sourceData = fs.readFileSync(source);
fs.appendFileSync(destination, `${prefixData}${sourceData}`);
} catch (err) {
logger.error('Problem copying results to output file', err);
}
}

export async function assessTocContents(
locations: ContainerResult['locations'],
schemaManager: SchemaManager,
Expand Down Expand Up @@ -230,7 +221,11 @@ export async function validateTocContents(
);
}
}
return [...providerReferences.values()];
if (providerReferences) {
return [...providerReferences.values()];
} else {
return [];
}
}

async function validateInNetworkFixedVersion(
Expand Down Expand Up @@ -518,24 +513,24 @@ export async function validateReferencedProviders(
}

export function writeIndexFile(urls: DockerManager['processedUrls'], outputDir: string) {
const schemaSizeTotals = new Map<string, number>();
const indexContents = urls
.map((record, index) => {
const indexPath = path.join(outputDir, 'result-index.txt');
try {
fs.writeFileSync(indexPath, '');
const schemaSizeTotals = new Map<string, number>();
urls.forEach((record, index) => {
const size = bytesToReadableSize(record.size);
schemaSizeTotals.set(record.schema, (schemaSizeTotals.get(record.schema) ?? 0) + record.size);
return `${index + 1}\t${record.schema}\t${record.uri}\t${size}`;
})
.join(os.EOL);
const sizeInfo = [...schemaSizeTotals.entries()]
.map(([schema, size]) => {
return `${schema}: ${bytesToReadableSize(size)}`;
})
.join(os.EOL);
logger.info(sizeInfo);
fs.writeFileSync(
path.join(outputDir, 'result-index.txt'),
`${indexContents}${os.EOL}${os.EOL}${sizeInfo}`
);
fs.appendFileSync(indexPath, `${index + 1}\t${record.schema}\t${record.uri}\t${size}`);
});
const sizeInfo = [...schemaSizeTotals.entries()]
.map(([schema, size]) => {
return `${schema}: ${bytesToReadableSize(size)}`;
})
.join(os.EOL);
fs.appendFileSync(indexPath, `${os.EOL}${os.EOL}${sizeInfo}`);
} catch (err) {
logger.error(err);
}
}

export function bytesToReadableSize(bytes: number): string {
Expand Down

0 comments on commit 9e087fb

Please sign in to comment.