From ea8728d5577be4e843d1e8a8fed373cf21ed1a75 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:57:11 +0100 Subject: [PATCH 1/8] Add option to generate hashes from list of files Use --generate (-g) option to consume [files] on command line as containers of lists of files to generate hashes for --- cli/xxhsum.c | 371 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 341 insertions(+), 30 deletions(-) diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 4c0d372d..49ffc02e 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -388,7 +388,15 @@ typedef enum { big_endian, little_endian} Display_endianness; typedef enum { display_gnu, display_bsd } Display_convention; -typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* line display signature */ +typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* filename display signature */ + +typedef enum { + LineStatus_hashOk, + LineStatus_hashFailed, + LineStatus_failedToOpen, + LineStatus_isDirectory, + LineStatus_memoryError +} LineStatus; static XSUM_displayLine_f XSUM_kDisplayLine_fTable[2][2] = { { XSUM_printLine_GNU, XSUM_printLine_GNU_LE }, @@ -414,13 +422,11 @@ static int XSUM_hashFile(const char* fileName, XSUM_setBinaryMode(stdin); } else { if (XSUM_isDirectory(fileName)) { - XSUM_log("xxhsum: %s: Is a directory \n", fileName); - return 1; + return LineStatus_isDirectory; } inFile = XSUM_fopen( fileName, "rb" ); if (inFile==NULL) { - XSUM_log("Error: Could not open '%s': %s. \n", fileName, strerror(errno)); - return 1; + return LineStatus_failedToOpen; } } /* Memory allocation & streaming */ @@ -428,7 +434,7 @@ static int XSUM_hashFile(const char* fileName, if (buffer == NULL) { XSUM_log("\nError: Out of memory.\n"); fclose(inFile); - return 1; + return LineStatus_memoryError; } /* Stream file & update hash */ @@ -469,7 +475,7 @@ static int XSUM_hashFile(const char* fileName, assert(0); /* not possible */ } - return 0; + return LineStatus_hashOk; } @@ -485,11 +491,47 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal, int fnNb; int result = 0; - if (fnTotal==0) - return XSUM_hashFile(stdinName, hashType, displayEndianness, convention); + if (fnTotal == 0) + { + int filestatus = XSUM_hashFile(stdinName, hashType, displayEndianness, convention); + switch (filestatus) + { + case LineStatus_isDirectory: + XSUM_log("xxhsum: %s: Is a directory \n", stdinName); + break; + case LineStatus_failedToOpen: + XSUM_log("Error: Could not open '%s': %s. \n", stdinName, strerror(errno)); + break; + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + } + + if (filestatus != LineStatus_hashOk) + result = 1; + } + + + for (fnNb = 0; fnNb < fnTotal; fnNb++) + { + int filestatus = XSUM_hashFile(fnList[fnNb], hashType, displayEndianness, convention); + switch (filestatus) + { + case LineStatus_isDirectory: + XSUM_log("xxhsum: %s: Is a directory \n", fnList[fnNb]); + break; + case LineStatus_failedToOpen: + XSUM_log("Error: Could not open '%s': %s. \n", fnList[fnNb], strerror(errno)); + break; + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + } + + if (filestatus != LineStatus_hashOk) + result = 1; + } - for (fnNb=0; fnNbfilename = NULL; + return ParseLine_invalidFormat; + } + } + + parsedLine->filename = filename; + + return ParseLine_ok; +} + +/* + * Parse gen source file. + */ +static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention) +{ + const char* const inFileName = XSUM_parseGenArg->inFileName; + ParseFileReport* const report = &XSUM_parseGenArg->report; + + unsigned long lineNumber = 0; + memset(report, 0, sizeof(*report)); + + ParsedLine parsedLine; + while (!report->quit) { + LineStatus lineStatus = LineStatus_hashFailed; + memset(&parsedLine, 0, sizeof(parsedLine)); + + lineNumber++; + if (lineNumber == 0) { + /* This is unlikely happen, but md5sum.c has this error check. */ + XSUM_log("%s: Error: Too many generate lines\n", inFileName); + report->quit = 1; + break; + } + + { GetLineResult const XSUM_getLineResult = XSUM_getLine(&XSUM_parseGenArg->lineBuf, + &XSUM_parseGenArg->lineMax, + XSUM_parseGenArg->inFile); + + /* Ignore comment lines */ + if (XSUM_getLineResult == GetLine_comment) { + continue; + } + + if (XSUM_getLineResult != GetLine_ok) { + if (XSUM_getLineResult == GetLine_eof) break; + + switch (XSUM_getLineResult) + { + case GetLine_ok: + case GetLine_comment: + case GetLine_eof: + /* These cases never happen. See above XSUM_getLineResult related "if"s. + They exist just for make gcc's -Wswitch-enum happy. */ + assert(0); + break; + + default: + XSUM_log("%s:%lu: Error: Unknown error.\n", inFileName, lineNumber); + break; + + case GetLine_exceedMaxLineLength: + XSUM_log("%s:%lu: Error: Line too long.\n", inFileName, lineNumber); + break; + + case GetLine_outOfMemory: + XSUM_log("%s:%lu: Error: Out of memory.\n", inFileName, lineNumber); + break; + } + report->quit = 1; + break; + } } + + if (XSUM_parseGenLine(&parsedLine, XSUM_parseGenArg->lineBuf) != ParseLine_ok) { + report->nImproperlyFormattedLines++; + if (XSUM_parseGenArg->warn) { + XSUM_log("%s:%lu: Error: Improperly formatted line.\n", + inFileName, lineNumber); + } + continue; + } + + report->nProperlyFormattedLines++; + + lineStatus = XSUM_hashFile(parsedLine.filename, hashType, displayEndianness, convention); + + switch (lineStatus) + { + default: + XSUM_log("%s: Error: Unknown error.\n", parsedLine.filename); + report->quit = 1; + break; + + case LineStatus_memoryError: + XSUM_log("\nError: Out of memory.\n"); + break; + + case LineStatus_failedToOpen: + case LineStatus_isDirectory: + if (XSUM_parseGenArg->ignoreMissing) { + report->nMissing++; + } + else { + report->nOpenOrReadFailures++; + if (!XSUM_parseGenArg->statusOnly) { + XSUM_output( + lineStatus == LineStatus_failedToOpen ? + "%s:%lu: Could not open or read '%s': %s.\n" : + "%s:%lu: Target is a directory '%s'.\n", // Leaves errno argument unconsumed + inFileName, lineNumber, parsedLine.filename, strerror(errno)); + } + } + break; + + case LineStatus_hashOk: + case LineStatus_hashFailed: + { int b = 1; + if (lineStatus == LineStatus_hashOk) { + /* If --quiet is specified, don't display "OK" */ + if (XSUM_parseGenArg->quiet) b = 0; + } + + if (b && !XSUM_parseGenArg->statusOnly) { + const int needsEscape = XSUM_filenameNeedsEscape(parsedLine.filename); + if (needsEscape) { + XSUM_output("%c", '\\'); + } + XSUM_printFilename(parsedLine.filename, needsEscape); + XSUM_output(": %s\n", lineStatus == LineStatus_hashOk ? "OK" : "FAILED"); + } } + break; + } + } /* while (!report->quit) */ +} + + +/* Parse text file for list of targets. + */ +static int XSUM_generateFile(const char* inFileName, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention, + XSUM_U32 strictMode, + XSUM_U32 statusOnly, + XSUM_U32 ignoreMissing, + XSUM_U32 warn, + XSUM_U32 quiet + ) +{ + int result = 0; + FILE* inFile = NULL; + ParseFileArg XSUM_parseGenArgBody; + ParseFileArg* const XSUM_parseGenArg = &XSUM_parseGenArgBody; + ParseFileReport* const report = &XSUM_parseGenArg->report; + + /* note: stdinName is special constant pointer. It is not a string. */ + if (inFileName == stdinName) { + /* + * Note: Since we expect text input for xxhash -c mode, + * we don't set binary mode for stdin. + */ + inFileName = stdinFileName; /* "stdin" */ + inFile = stdin; + } + else { + inFile = XSUM_fopen(inFileName, "rt"); + } + + if (inFile == NULL) { + XSUM_log("Error: Could not open '%s': %s\n", inFileName, strerror(errno)); + return 0; + } + + XSUM_parseGenArg->inFileName = inFileName; + XSUM_parseGenArg->inFile = inFile; + XSUM_parseGenArg->lineMax = DEFAULT_LINE_LENGTH; + XSUM_parseGenArg->lineBuf = (char*)malloc((size_t)XSUM_parseGenArg->lineMax); + XSUM_parseGenArg->blockSize = 64 * 1024; + XSUM_parseGenArg->blockBuf = (char*)malloc(XSUM_parseGenArg->blockSize); + XSUM_parseGenArg->statusOnly = statusOnly; + XSUM_parseGenArg->ignoreMissing = ignoreMissing; + XSUM_parseGenArg->warn = warn; + XSUM_parseGenArg->quiet = quiet; + + if ((XSUM_parseGenArg->lineBuf == NULL) + || (XSUM_parseGenArg->blockBuf == NULL)) { + XSUM_log("Error: : memory allocation failed \n"); + exit(1); + } + XSUM_parseGenFile1(XSUM_parseGenArg, hashType, displayEndianness, convention); + + free(XSUM_parseGenArg->blockBuf); + free(XSUM_parseGenArg->lineBuf); + + if (inFile != stdin) fclose(inFile); + + /* Show error/warning messages. All messages are copied from md5sum.c + */ + if (report->nProperlyFormattedLines == 0) { + XSUM_log("%s: no properly formatted filename lines found\n", inFileName); + } + if (report->nImproperlyFormattedLines) { + XSUM_output("%lu %s improperly formatted\n" + , report->nImproperlyFormattedLines + , report->nImproperlyFormattedLines == 1 ? "line is" : "lines are"); + } + if (report->nOpenOrReadFailures) { + XSUM_output("%lu listed %s could not be read\n" + , report->nOpenOrReadFailures + , report->nOpenOrReadFailures == 1 ? "file" : "files"); + } + /* Result (exit) code logic is copied from + * gnu coreutils/src/md5sum.c digest_check() */ + result = report->nProperlyFormattedLines != 0 + && report->nOpenOrReadFailures == 0 + && (report->nImproperlyFormattedLines == 0) + && report->quit == 0; + + return result; +} + +static int XSUM_generateFiles(const char* fnList[], int fnTotal, + AlgoSelected hashType, + Display_endianness displayEndianness, + Display_convention convention, + XSUM_U32 strictMode, + XSUM_U32 statusOnly, + XSUM_U32 ignoreMissing, + XSUM_U32 warn, + XSUM_U32 quiet) +{ + int ok = 1; + + /* Special case for stdinName "-", + * note: stdinName is not a string. It's special pointer. */ + if (fnTotal == 0) { + ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + } + else { + int fnNb; + for (fnNb = 0; fnNb < fnTotal; fnNb++) + ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + } + return ok ? 0 : 1; +} + + /* ******************************************************** * Main **********************************************************/ @@ -1101,17 +1402,18 @@ static int XSUM_checkFiles(const char* fnList[], int fnTotal, static int XSUM_usage(const char* exename) { XSUM_log( WELCOME_MESSAGE(exename) ); - XSUM_log( "Print or verify checksums using fast non-cryptographic algorithm xxHash \n\n" ); + XSUM_log( "Create or verify checksums using fast non-cryptographic algorithm xxHash \n\n" ); XSUM_log( "Usage: %s [options] [files] \n\n", exename); XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n"); XSUM_log( "\nOptions: \n"); - XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); - XSUM_log( " 0: XXH32 \n"); - XSUM_log( " 1: XXH64 \n"); - XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); - XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); - XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); - XSUM_log( " -h, --help display a long help page about advanced options \n"); + XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); + XSUM_log( " 0: XXH32 \n"); + XSUM_log( " 1: XXH64 \n"); + XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); + XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); + XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); + XSUM_log( " -g, --generate generate hashes for files listed in [files] \n"); + XSUM_log( " -h, --help display a long help page about advanced options \n"); return 0; } @@ -1129,11 +1431,11 @@ static int XSUM_usage_advanced(const char* exename) XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT); XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n"); XSUM_log( "\n"); - XSUM_log( "The following five options are useful only when verifying checksums (-c): \n"); - XSUM_log( " -q, --quiet Don't print OK for each successfully verified file \n"); + XSUM_log( "The following five options are useful only when using lists in [files] to verify or generate checksums (-c): \n"); + XSUM_log( " -q, --quiet Don't print OK for each successfully processed file \n"); XSUM_log( " --status Don't output anything, status code shows success \n"); - XSUM_log( " --strict Exit non-zero for improperly formatted checksum lines \n"); - XSUM_log( " --warn Warn about improperly formatted checksum lines \n"); + XSUM_log( " --strict Exit non-zero for improperly formatted lines in [files] \n"); + XSUM_log( " --warn Warn about improperly formatted lines in [files] \n"); XSUM_log( " --ignore-missing Don't fail or report status for missing files \n"); return 0; } @@ -1239,6 +1541,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) assert(argument != NULL); if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; } + if (!strcmp(argument, "--generate")) { fileCheckMode = 2; continue; } if (!strcmp(argument, "--benchmark-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--bench-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--quiet")) { XSUM_logLevel--; continue; } @@ -1300,6 +1603,12 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) argument++; break; + /* Generate hash mode */ + case 'g': + fileCheckMode = 2; + argument++; + break; + /* Warning mode (file check mode only, alias of "--warning") */ case 'w': warn=1; @@ -1361,9 +1670,11 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_badusage(exename); if (filenamesStart==0) filenamesStart = argc; - if (fileCheckMode) { + if (fileCheckMode == 1) { return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); + } else if (fileCheckMode == 2) { + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); } From b9f37d9948d938b2e921d1b61837e806daf15bf0 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:46:29 +0100 Subject: [PATCH 2/8] Fix CI bugs --- cli/xxhsum.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 49ffc02e..534a8a45 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -956,6 +956,11 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev) report->quit = 1; break; + case LineStatus_memoryError: + case LineStatus_isDirectory: + assert(0); /* Never happens on these paths */ + break; + case LineStatus_failedToOpen: if (XSUM_parseFileArg->ignoreMissing) { report->nMissing++; @@ -1142,7 +1147,8 @@ static ParseLineResult XSUM_parseGenLine(ParsedLine * parsedLine, if (XSUM_lineNeedsUnescape(filename)) { ++filename; - const size_t filenameLen = strlen(filename); + size_t filenameLen; + filenameLen = strlen(filename); char* const result = XSUM_filenameUnescape(filename, filenameLen); if (result == NULL) { @@ -1170,9 +1176,9 @@ static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg, unsigned long lineNumber = 0; memset(report, 0, sizeof(*report)); - ParsedLine parsedLine; while (!report->quit) { LineStatus lineStatus = LineStatus_hashFailed; + ParsedLine parsedLine; memset(&parsedLine, 0, sizeof(parsedLine)); lineNumber++; @@ -1256,7 +1262,7 @@ static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg, XSUM_output( lineStatus == LineStatus_failedToOpen ? "%s:%lu: Could not open or read '%s': %s.\n" : - "%s:%lu: Target is a directory '%s'.\n", // Leaves errno argument unconsumed + "%s:%lu: Target is a directory '%s'.\n", /* Leaves errno argument unconsumed */ inFileName, lineNumber, parsedLine.filename, strerror(errno)); } } @@ -1290,7 +1296,6 @@ static int XSUM_generateFile(const char* inFileName, AlgoSelected hashType, Display_endianness displayEndianness, Display_convention convention, - XSUM_U32 strictMode, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, XSUM_U32 warn, @@ -1376,20 +1381,19 @@ static int XSUM_generateFiles(const char* fnList[], int fnTotal, XSUM_U32 strictMode, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, - XSUM_U32 warn, - XSUM_U32 quiet) + XSUM_U32 warn) { int ok = 1; /* Special case for stdinName "-", * note: stdinName is not a string. It's special pointer. */ if (fnTotal == 0) { - ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); } else { int fnNb; for (fnNb = 0; fnNb < fnTotal; fnNb++) - ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); } return ok ? 0 : 1; } @@ -1674,7 +1678,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); } else if (fileCheckMode == 2) { - return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); } From 037e8a72574b3b2aa9964ed22bafa4a414a7dcf7 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Tue, 8 Oct 2024 18:19:26 +0100 Subject: [PATCH 3/8] Fix test build errors --- cli/xxhsum.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 534a8a45..8436f2c1 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -403,7 +403,7 @@ static XSUM_displayLine_f XSUM_kDisplayLine_fTable[2][2] = { { XSUM_printLine_BSD, XSUM_printLine_BSD_LE } }; -static int XSUM_hashFile(const char* fileName, +static LineStatus XSUM_hashFile(const char* fileName, const AlgoSelected hashType, const Display_endianness displayEndianness, const Display_convention convention) @@ -493,20 +493,26 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal, if (fnTotal == 0) { - int filestatus = XSUM_hashFile(stdinName, hashType, displayEndianness, convention); + LineStatus filestatus = XSUM_hashFile(stdinName, hashType, displayEndianness, convention); switch (filestatus) { + case LineStatus_hashOk: + case LineStatus_hashFailed: + break; case LineStatus_isDirectory: XSUM_log("xxhsum: %s: Is a directory \n", stdinName); + result = 1; break; case LineStatus_failedToOpen: XSUM_log("Error: Could not open '%s': %s. \n", stdinName, strerror(errno)); + result = 1; break; case LineStatus_memoryError: XSUM_log("\nError: Out of memory.\n"); + result = 1; break; } - + if (filestatus != LineStatus_hashOk) result = 1; } @@ -514,9 +520,12 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal, for (fnNb = 0; fnNb < fnTotal; fnNb++) { - int filestatus = XSUM_hashFile(fnList[fnNb], hashType, displayEndianness, convention); + LineStatus filestatus = XSUM_hashFile(fnList[fnNb], hashType, displayEndianness, convention); switch (filestatus) { + case LineStatus_hashOk: + case LineStatus_hashFailed: + break; case LineStatus_isDirectory: XSUM_log("xxhsum: %s: Is a directory \n", fnList[fnNb]); break; @@ -1145,13 +1154,11 @@ static ParseLineResult XSUM_parseGenLine(ParsedLine * parsedLine, char* filename) { if (XSUM_lineNeedsUnescape(filename)) { - ++filename; - size_t filenameLen; + ++filename; filenameLen = strlen(filename); - char* const result = XSUM_filenameUnescape(filename, filenameLen); - if (result == NULL) { + if (XSUM_filenameUnescape(filename, filenameLen) == NULL) { parsedLine->filename = NULL; return ParseLine_invalidFormat; } @@ -1378,7 +1385,6 @@ static int XSUM_generateFiles(const char* fnList[], int fnTotal, AlgoSelected hashType, Display_endianness displayEndianness, Display_convention convention, - XSUM_U32 strictMode, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, XSUM_U32 warn) @@ -1678,7 +1684,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); } else if (fileCheckMode == 2) { - return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, strictMode, statusOnly, ignoreMissing, warn); + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); } From 181e065e55f92e2b5f0b6e92c01cf9b744235036 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Tue, 8 Oct 2024 19:39:49 +0100 Subject: [PATCH 4/8] Mainly fixing erroneous whitespace --- cli/xxhsum.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 8436f2c1..51bc1e7f 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -388,7 +388,7 @@ typedef enum { big_endian, little_endian} Display_endianness; typedef enum { display_gnu, display_bsd } Display_convention; -typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* filename display signature */ +typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* line display signature */ typedef enum { LineStatus_hashOk, @@ -501,18 +501,15 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal, break; case LineStatus_isDirectory: XSUM_log("xxhsum: %s: Is a directory \n", stdinName); - result = 1; break; case LineStatus_failedToOpen: XSUM_log("Error: Could not open '%s': %s. \n", stdinName, strerror(errno)); - result = 1; break; case LineStatus_memoryError: XSUM_log("\nError: Out of memory.\n"); - result = 1; break; } - + if (filestatus != LineStatus_hashOk) result = 1; } @@ -1165,7 +1162,7 @@ static ParseLineResult XSUM_parseGenLine(ParsedLine * parsedLine, } parsedLine->filename = filename; - + return ParseLine_ok; } @@ -1387,19 +1384,20 @@ static int XSUM_generateFiles(const char* fnList[], int fnTotal, Display_convention convention, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, - XSUM_U32 warn) + XSUM_U32 warn, + XSUM_U32 quiet) { int ok = 1; /* Special case for stdinName "-", * note: stdinName is not a string. It's special pointer. */ if (fnTotal == 0) { - ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, quiet); } else { int fnNb; for (fnNb = 0; fnNb < fnTotal; fnNb++) - ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, quiet); } return ok ? 0 : 1; } @@ -1684,7 +1682,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); } else if (fileCheckMode == 2) { - return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn); + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); } From a923d8f25339a27e8aa9052be4d2c02f81876c4c Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:25:43 +0100 Subject: [PATCH 5/8] Add --generate to man page Describe the use and rationale for the --generate option in the man page --- cli/xxhsum.1.md | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/cli/xxhsum.1.md b/cli/xxhsum.1.md index c66b6b93..dabdaab1 100644 --- a/cli/xxhsum.1.md +++ b/cli/xxhsum.1.md @@ -5,6 +5,7 @@ SYNOPSIS -------- `xxhsum` [*OPTION*]... [*FILE*]... + `xxhsum -b` [*OPTION*]... `xxh32sum` is equivalent to `xxhsum -H0`, @@ -56,16 +57,23 @@ OPTIONS * `-h`, `--help`: Displays help and exits -### The following options are useful only when verifying checksums (-c): +### The following options are useful only when using lists in *FILE* (-c and -g): * `-c`, `--check` *FILE*: Read xxHash sums from *FILE* and check them +* `-g`, `--generate` *FILE*: + Read filenames from *FILE* and generate hashes for them. + Valid *FILE*s have one filename per line, which can include embedded spaces, etc with no need for quotes, escapes, etc. + Note that a line commencing with '\\' will enable the convention used in the encoding of filenames against output hashes, + whereby subsequent \\\\, \n and \r seqeuences are converted to the single + character 0x5C, 0x0A and 0x0D respectively. + * `-q`, `--quiet`: - Don't print OK for each successfully verified file + Don't print OK for each successfully processed line of *FILE* * `--strict`: - Return an error code if any line in the file is invalid, + Return an error code if any line in *FILE** is invalid, not just if some checksums are wrong. This policy is disabled by default, though UI will prompt an informational message @@ -75,7 +83,7 @@ OPTIONS Don't output anything. Status code shows success. * `-w`, `--warn`: - Emit a warning message about each improperly formatted checksum line. + Emit a warning message about each improperly formatted line in *FILE*. ### The following options are useful only benchmark purpose: @@ -119,6 +127,19 @@ Read xxHash sums from specific files and check them $ xxhsum -c xyz.xxh32 qux.xxh64 +Produce a list of files, then generate hashes for that list + + $ find . -type f -name '*.[ch]' > c-files.txt + $ xxhsum --quiet -g c-files.txt + +Read the list of files from standard input to avoid the need for an intermediate file + + $ find . -type f -name '*.[ch]' | xxhsum --quiet -g - + +Note that if shell expansion, length of argument list, clarity of use of spaces in filenames, etc allow it then the same output as the previous example can be generated like this + + $ xxhsum `find . -name '*.[ch]'` + Benchmark xxHash algorithm. By default, `xxhsum` benchmarks xxHash main variants on a synthetic sample of 100 KB, From d033aefba975c594726a21917ab51f6bb14780e4 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Wed, 9 Oct 2024 12:28:44 +0100 Subject: [PATCH 6/8] Add git options for Visual Studio Fix two snags: - Bug / quirk of Visual Studio causes failure to process CMakeLists.txt if .gitignore contains *.txt - .vs/ folder artefacts are not to be controlled --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e6473396..1493a339 100644 --- a/.gitignore +++ b/.gitignore @@ -47,12 +47,14 @@ tmp* tests/*.unicode tests/unicode_test* *.txt +!CMakeLists.txt *.xxhsum # editor artifacts .clang_complete *.swp .vscode/ +.vs/ # Doxygen doxygen/ From 68ea87f46a90fe4b331b295bcc130dfca894d868 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:04:09 +0100 Subject: [PATCH 7/8] Revert "Add git options for Visual Studio" This reverts commit d033aefba975c594726a21917ab51f6bb14780e4. --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index 1493a339..e6473396 100644 --- a/.gitignore +++ b/.gitignore @@ -47,14 +47,12 @@ tmp* tests/*.unicode tests/unicode_test* *.txt -!CMakeLists.txt *.xxhsum # editor artifacts .clang_complete *.swp .vscode/ -.vs/ # Doxygen doxygen/ From 40511abc815a4f755df097c58ec43b7f5a2e4384 Mon Sep 17 00:00:00 2001 From: Ian-Clowes <65678428+Ian-Clowes@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:56:33 +0100 Subject: [PATCH 8/8] Tidy up parameter usage - Use --files-from instead of --generate for new feature - Remove the need for --quiet when generating hashes from file list --- cli/xxhsum.1.md | 10 ++++----- cli/xxhsum.c | 56 +++++++++++++++++-------------------------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/cli/xxhsum.1.md b/cli/xxhsum.1.md index dabdaab1..8f005c28 100644 --- a/cli/xxhsum.1.md +++ b/cli/xxhsum.1.md @@ -57,12 +57,12 @@ OPTIONS * `-h`, `--help`: Displays help and exits -### The following options are useful only when using lists in *FILE* (-c and -g): +### The following options are useful only when using lists in *FILE* (--check and --files-from): * `-c`, `--check` *FILE*: Read xxHash sums from *FILE* and check them -* `-g`, `--generate` *FILE*: +* `--files-from` *FILE*: Read filenames from *FILE* and generate hashes for them. Valid *FILE*s have one filename per line, which can include embedded spaces, etc with no need for quotes, escapes, etc. Note that a line commencing with '\\' will enable the convention used in the encoding of filenames against output hashes, @@ -70,7 +70,7 @@ OPTIONS character 0x5C, 0x0A and 0x0D respectively. * `-q`, `--quiet`: - Don't print OK for each successfully processed line of *FILE* + Don't print OK for each successfully verified hash (only for --check) * `--strict`: Return an error code if any line in *FILE** is invalid, @@ -130,11 +130,11 @@ Read xxHash sums from specific files and check them Produce a list of files, then generate hashes for that list $ find . -type f -name '*.[ch]' > c-files.txt - $ xxhsum --quiet -g c-files.txt + $ xxhsum --files-from c-files.txt Read the list of files from standard input to avoid the need for an intermediate file - $ find . -type f -name '*.[ch]' | xxhsum --quiet -g - + $ find . -type f -name '*.[ch]' | xxhsum --files-from - Note that if shell expansion, length of argument list, clarity of use of spaces in filenames, etc allow it then the same output as the previous example can be generated like this diff --git a/cli/xxhsum.c b/cli/xxhsum.c index 51bc1e7f..dfa41efd 100644 --- a/cli/xxhsum.c +++ b/cli/xxhsum.c @@ -1274,20 +1274,6 @@ static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg, case LineStatus_hashOk: case LineStatus_hashFailed: - { int b = 1; - if (lineStatus == LineStatus_hashOk) { - /* If --quiet is specified, don't display "OK" */ - if (XSUM_parseGenArg->quiet) b = 0; - } - - if (b && !XSUM_parseGenArg->statusOnly) { - const int needsEscape = XSUM_filenameNeedsEscape(parsedLine.filename); - if (needsEscape) { - XSUM_output("%c", '\\'); - } - XSUM_printFilename(parsedLine.filename, needsEscape); - XSUM_output(": %s\n", lineStatus == LineStatus_hashOk ? "OK" : "FAILED"); - } } break; } } /* while (!report->quit) */ @@ -1302,9 +1288,7 @@ static int XSUM_generateFile(const char* inFileName, Display_convention convention, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, - XSUM_U32 warn, - XSUM_U32 quiet - ) + XSUM_U32 warn) { int result = 0; FILE* inFile = NULL; @@ -1339,7 +1323,6 @@ static int XSUM_generateFile(const char* inFileName, XSUM_parseGenArg->statusOnly = statusOnly; XSUM_parseGenArg->ignoreMissing = ignoreMissing; XSUM_parseGenArg->warn = warn; - XSUM_parseGenArg->quiet = quiet; if ((XSUM_parseGenArg->lineBuf == NULL) || (XSUM_parseGenArg->blockBuf == NULL)) { @@ -1384,20 +1367,19 @@ static int XSUM_generateFiles(const char* fnList[], int fnTotal, Display_convention convention, XSUM_U32 statusOnly, XSUM_U32 ignoreMissing, - XSUM_U32 warn, - XSUM_U32 quiet) + XSUM_U32 warn) { int ok = 1; /* Special case for stdinName "-", * note: stdinName is not a string. It's special pointer. */ if (fnTotal == 0) { - ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, quiet); + ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn); } else { int fnNb; for (fnNb = 0; fnNb < fnTotal; fnNb++) - ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn, quiet); + ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn); } return ok ? 0 : 1; } @@ -1414,14 +1396,14 @@ static int XSUM_usage(const char* exename) XSUM_log( "Usage: %s [options] [files] \n\n", exename); XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n"); XSUM_log( "\nOptions: \n"); - XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); - XSUM_log( " 0: XXH32 \n"); - XSUM_log( " 1: XXH64 \n"); - XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); - XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); - XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); - XSUM_log( " -g, --generate generate hashes for files listed in [files] \n"); - XSUM_log( " -h, --help display a long help page about advanced options \n"); + XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo); + XSUM_log( " 0: XXH32 \n"); + XSUM_log( " 1: XXH64 \n"); + XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n"); + XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n"); + XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n"); + XSUM_log( " --files-from generate hashes for files listed in [files] \n"); + XSUM_log( " -h, --help display a long help page about advanced options \n"); return 0; } @@ -1439,8 +1421,8 @@ static int XSUM_usage_advanced(const char* exename) XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT); XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n"); XSUM_log( "\n"); - XSUM_log( "The following five options are useful only when using lists in [files] to verify or generate checksums (-c): \n"); - XSUM_log( " -q, --quiet Don't print OK for each successfully processed file \n"); + XSUM_log( "The following five options are useful only when using lists in [files] to verify or generate checksums: \n"); + XSUM_log( " -q, --quiet Don't print OK for each successfully verified hash \n"); XSUM_log( " --status Don't output anything, status code shows success \n"); XSUM_log( " --strict Exit non-zero for improperly formatted lines in [files] \n"); XSUM_log( " --warn Warn about improperly formatted lines in [files] \n"); @@ -1549,7 +1531,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) assert(argument != NULL); if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; } - if (!strcmp(argument, "--generate")) { fileCheckMode = 2; continue; } + if (!strcmp(argument, "--files-from")) { fileCheckMode = 2; continue; } if (!strcmp(argument, "--benchmark-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--bench-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; } if (!strcmp(argument, "--quiet")) { XSUM_logLevel--; continue; } @@ -1611,11 +1593,11 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) argument++; break; - /* Generate hash mode */ - case 'g': + /* Generate hash mode (tar style short form of --files-from) + case 'T': fileCheckMode = 2; argument++; - break; + break; */ /* Warning mode (file check mode only, alias of "--warning") */ case 'w': @@ -1682,7 +1664,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[]) return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask); } else if (fileCheckMode == 2) { - return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/); + return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn); } else { return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention); }