diff --git a/Makefile b/Makefile index 11db9d7a..62778aaf 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ -# xxhsum : provides 32/64 bits hash of a file, or piped data +# xxhsum : provides 32/64 bits hash of one or multiple files, or stdin # ################################################################ CFLAGS ?= -O3 @@ -51,9 +51,15 @@ xxhsum32: xxhash.c xxhsum.c $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) test: clean xxhsum + # stdin ./xxhsum < xxhash.c + # multiple files + ./xxhsum * + # internal bench ./xxhsum -bi1 + # file bench ./xxhsum -bi1 xxhash.c + # memory tests valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c diff --git a/README.md b/README.md index 406aa4ff..b36428ed 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,17 @@ The reference system uses a Core i5-3340M @2.7GHz | XXH64 | 13.8 GB/s | 1.9 GB/s | | XXH32 | 6.8 GB/s | 6.0 GB/s | + +### License + +The library files `xxhash.c` and `xxhash.h` are BSD licensed. +The utility `xxhsum` is GPL licensed. + + +### Other languages + Beyond the C reference version, xxHash is also available on many programming languages, thanks to great contributors. -They are [listed here](https://code.google.com/p/xxhash/). \ No newline at end of file +They are [listed here](https://code.google.com/p/xxhash/). + diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt new file mode 100644 index 00000000..75fca0de --- /dev/null +++ b/cmake_unofficial/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 2.6) +cmake_policy(VERSION 2.6) + +project(xxhash) + +set(XXHASH_LIB_VERSION "0.41.0") +set(XXHASH_LIB_SOVERSION "0") + +set(BUILD_STATIC_LIBS ON CACHE BOOL "Set to ON to build static libraries") +if(BUILD_STATIC_LIBS) + add_library(xxhashstatic ../xxhash.c) + set_target_properties(xxhashstatic PROPERTIES OUTPUT_NAME xxhash) +endif(BUILD_STATIC_LIBS) + +add_library(xxhash SHARED ../xxhash.c) +set_target_properties(xxhash PROPERTIES + COMPILE_DEFINITIONS "XXHASH_EXPORT" + VERSION "${XXHASH_LIB_VERSION}" + SOVERSION "${XXHASH_LIB_SOVERSION}") + diff --git a/xxhash.c b/xxhash.c index e6fb8f14..a18b9785 100644 --- a/xxhash.c +++ b/xxhash.c @@ -35,13 +35,15 @@ You can contact the author at : /************************************** * Tuning parameters **************************************/ -/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. - * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +/* XXH_FORCE_DIRECT_UNALIGNED_MEMORY_ACCESS + * Unaligned memory access is automatically enabled for "common" CPU, such as x86/x64. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure proper working with unaligned memory accesses. + * If you know your target CPU efficiently supports unaligned memory accesses, you can force this option manually. + * If your CPU efficiently supports unaligned memory accesses and the compiler did not automatically detected it, you will witness large performance improvement. + * You can also enable this switch from compilation command line / Makefile. */ -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 +#if !defined(XXH_FORCE_DIRECT_MEMORY_ACCESS) && ( defined(__ARM_FEATURE_UNALIGNED) ) +# define XXH_FORCE_DIRECT_MEMORY_ACCESS 1 #endif /* XXH_ACCEPT_NULL_INPUT_POINTER : @@ -61,6 +63,15 @@ You can contact the author at : */ #define XXH_FORCE_NATIVE_FORMAT 0 +/* XXH_USELESS_ALIGN_BRANCH : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : don't make a test between aligned/unaligned, because performance will be the same. + * It avoids one initial branch per hash. + */ +#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) || defined(XXH_FORCE_DIRECT_MEMORY_ACCESS) +# define XXH_USELESS_ALIGN_BRANCH 1 +#endif + /************************************** * Compiler Specific Options @@ -113,20 +124,29 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp typedef unsigned long long U64; #endif + +#if defined(XXH_FORCE_DIRECT_MEMORY_ACCESS) + +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#else + static U32 XXH_read32(const void* memPtr) { - U32 val32; - memcpy(&val32, memPtr, 4); - return val32; + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; } static U64 XXH_read64(const void* memPtr) { - U64 val64; - memcpy(&val64, memPtr, 8); - return val64; + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; } +#endif // defined /****************************************** @@ -175,8 +195,10 @@ static U64 XXH_swap64 (U64 x) * Architecture Macros ***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ -static const int one = 1; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example one the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int one = 1; # define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) #endif @@ -315,7 +337,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned XXH32 (const void* input, size_t len, unsigned seed) +unsigned int XXH32 (const void* input, size_t len, unsigned int seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ @@ -326,7 +348,7 @@ unsigned XXH32 (const void* input, size_t len, unsigned seed) #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; -# if !defined(XXH_USE_UNALIGNED_ACCESS) +# if !defined(XXH_USELESS_ALIGN_BRANCH) if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) @@ -466,7 +488,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; -# if !defined(XXH_USE_UNALIGNED_ACCESS) +# if !defined(XXH_USELESS_ALIGN_BRANCH) if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) @@ -538,7 +560,7 @@ XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) /*** Hash feed ***/ -XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, unsigned int seed) { XXH_istate32_t* state = (XXH_istate32_t*) state_in; state->seed = seed; @@ -708,7 +730,7 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endiane } -U32 XXH32_digest (const XXH32_state_t* state_in) +unsigned int XXH32_digest (const XXH32_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; diff --git a/xxhsum.c b/xxhsum.c index f9cd9b83..faaac769 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -136,6 +136,7 @@ static unsigned g_displayLevel = 1; *************************************/ static int g_nbIterations = NBLOOPS; static int g_fn_selection = 1; /* required within main() & usage() */ +static size_t g_sampleSize = 100 KB; /************************************* @@ -204,7 +205,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) } -static U64 BMK_GetFileSize(char* infilename) +static U64 BMK_GetFileSize(const char* infilename) { int r; #if defined(_MSC_VER) @@ -237,7 +238,7 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) int nbHashes = 0; int milliTime; - DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH32", (int)bufferSize); + DISPLAY("%1i-%-17.17s : %10i ->\r", iterationNb, "XXH32", (int)bufferSize); /* Timing loop */ milliTime = BMK_GetMilliStart(); @@ -254,24 +255,25 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestH*nbHashes) fastestH = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", iterationNb, "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + DISPLAY("%1i-%-17.17s : %10i -> %7.1f MB/s\r", iterationNb, "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000.); } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000., hashResult); + DISPLAY("%-19.19s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000., hashResult); } /* Bench XXH32 on Unaligned input */ + if (bufferSize>1) { int iterationNb; double fastestH = 100000000.; DISPLAY("\r%79s\r", ""); /* Clean display line */ - for (iterationNb = 1; (iterationNb <= g_nbIterations) && ((bufferSize>1)); iterationNb++) + for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) { int nbHashes = 0; int milliTime; const char* charPtr = (const char*)buffer; - DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "(unaligned)", (int)(bufferSize-1)); + DISPLAY("%1i-%-17.17s : %10i ->\r", iterationNb, "(unaligned)", (int)(bufferSize-1)); /* timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); @@ -287,9 +289,9 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestH*nbHashes) fastestH = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", iterationNb, "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + DISPLAY("%1i-%-17.17s : %10i -> %7.1f MB/s\r", iterationNb, "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + DISPLAY("%-19.19s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); } /* Bench XXH64 */ @@ -304,7 +306,7 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) int nbHashes = 0; int milliTime; - DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH64", (int)bufferSize); + DISPLAY("%1i-%-17.17s : %10i ->\r", iterationNb, "XXH64", (int)bufferSize); /* Timing loop */ milliTime = BMK_GetMilliStart(); @@ -321,10 +323,10 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) } milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestH*nbHashes) fastestH = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + DISPLAY("%1i-%-17.17s : %10i -> %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); } { - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + DISPLAY("%-19.19s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); DISPLAY("%08X%08X", (U32)(h64 >> 32), (U32)h64); DISPLAY("\n"); } @@ -332,14 +334,14 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) } -static int BMK_benchFile(char** fileNamesTable, int nbFiles) +static int BMK_benchFiles(const char** fileNamesTable, int nbFiles) { int fileIdx=0; while (fileIdx= XXH32_state_t */ /* Check file existence */ if (fileName == stdinName) @@ -558,7 +560,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) } /* Memory allocation & restrictions */ - buffer = (char*)malloc(blockSize); + buffer = malloc(blockSize); if(!buffer) { DISPLAY("\nError: not enough memory!\n"); @@ -611,14 +613,14 @@ static int BMK_hash(const char* fileName, U32 hashNb) { U32 h32 = XXH32_digest((XXH32_state_t*)&state); BMK_display_BigEndian(&h32, 4); - DISPLAYRESULT(" %s \n", fileName); + DISPLAYRESULT(" %s \n", fileName); break; } case 1: { U64 h64 = XXH64_digest(&state); BMK_display_BigEndian(&h64, 8); - DISPLAYRESULT(" %s \n", fileName); + DISPLAYRESULT(" %s \n", fileName); break; } default: @@ -629,6 +631,23 @@ static int BMK_hash(const char* fileName, U32 hashNb) } +static int BMK_hashFiles(const char** fnList, int fnTotal, U32 hashNb) +{ + int fnNb; + int result = 0; + if (fnTotal==0) + { + result = BMK_hash(stdinName, hashNb); + } + else + { + for (fnNb=0; fnNb='0' && argument[0]<='9') + g_sampleSize *= 10, g_sampleSize += argument[0]-'0', argument++; + break; + default: return badusage(exename); } @@ -720,13 +745,14 @@ int main(int argc, char** argv) DISPLAY( WELCOME_MESSAGE ); BMK_sanityCheck(); if (filenamesStart==0) return BMK_benchInternal(); - return BMK_benchFile(argv+filenamesStart, argc-filenamesStart); + return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart); } /* Check if input is defined as console; trigger an error in this case */ - if ((input_filename == stdinName) && IS_CONSOLE(stdin) ) return badusage(exename); + if ( (filenamesStart==0) && IS_CONSOLE(stdin) ) return badusage(exename); if(g_fn_selection < 0 || g_fn_selection > 1) return badusage(exename); - return BMK_hash(input_filename, g_fn_selection); + if (filenamesStart==0) filenamesStart = argc; + return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, g_fn_selection); }