From 86fe903d97b02a92e09ab5313b8e4c5b36eb8e65 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 9 Dec 2014 18:42:33 +0100 Subject: [PATCH 01/13] xxhsum display result using big-endian convention --- xxhsum.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/xxhsum.c b/xxhsum.c index e090e511..dcacb803 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -492,6 +492,11 @@ static void BMK_sanityCheck(void) DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } +static void BMK_display_BigEndian(const void* ptr, size_t length) +{ + const BYTE* p = ptr; + while (length--) DISPLAYRESULT("%02x", *p++); +} static int BMK_hash(const char* fileName, U32 hashNb) { @@ -568,13 +573,15 @@ static int BMK_hash(const char* fileName, U32 hashNb) case 0: { U32 h32 = XXH32_digest((XXH32_state_t*)&state); - DISPLAYRESULT("%08x %s \n", h32, fileName); + BMK_display_BigEndian(&h32, 4); + DISPLAYRESULT(" %s \n", fileName); break; } case 1: { U64 h64 = XXH64_digest(&state); - DISPLAYRESULT("%08x%08x %s \n", (U32)(h64>>32), (U32)(h64), fileName); + BMK_display_BigEndian(&h64, 8); + DISPLAYRESULT(" %s \n", fileName); break; } default: From bdbf0a541a315cce0c0d4cefb128a8bb1d847744 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 9 Dec 2014 19:16:15 +0100 Subject: [PATCH 02/13] minor comment update --- xxhash.c | 5 +++-- xxhash.h | 7 +++++-- xxhsum.c | 7 ++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/xxhash.c b/xxhash.c index 24a64b5f..9a54c25c 100644 --- a/xxhash.c +++ b/xxhash.c @@ -1,6 +1,6 @@ /* xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. +Copyright (C) 2012-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,7 +27,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ +- xxHash source repository : http://code.google.com/p/xxhash +- xxHash source mirror : https://github.com/Cyan4973/xxHash - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ diff --git a/xxhash.h b/xxhash.h index 55b45015..be0acced 100644 --- a/xxhash.h +++ b/xxhash.h @@ -1,7 +1,8 @@ /* xxHash - Extremely Fast Hash algorithm Header File - Copyright (C) 2012-2014, Yann Collet. + Copyright (C) 2012-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ + - xxHash source repository : http://code.google.com/p/xxhash + - xxHash source mirror : https://github.com/Cyan4973/xxHash + - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ /* Notice extracted from xxHash homepage : diff --git a/xxhsum.c b/xxhsum.c index dcacb803..3cf524c2 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -1,6 +1,6 @@ /* bench.c - Demo program to benchmark open-source algorithm -Copyright (C) Yann Collet 2012-2014 +Copyright (C) Yann Collet 2012-2015 This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,8 +17,9 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : -- Blog homepage : http://fastcompression.blogspot.com/ -- Discussion group : https://groups.google.com/forum/?fromgroups#!forum/lz4c +- xxHash source repository : http://code.google.com/p/xxhash +- xxHash source mirror : https://github.com/Cyan4973/xxHash +- public discussion board : https://groups.google.com/forum/#!forum/lz4c */ /************************************** From edb6d79f512c9eccf6c33127395cd1312377eb47 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Dec 2014 12:15:03 +0100 Subject: [PATCH 03/13] xxhash.c is C90 compatible --- xxhash.c | 164 ++++++++++++++++++++++++++++--------------------------- xxhsum.c | 10 ++-- 2 files changed, 90 insertions(+), 84 deletions(-) diff --git a/xxhash.c b/xxhash.c index 9a54c25c..a3ee31c0 100644 --- a/xxhash.c +++ b/xxhash.c @@ -1,6 +1,7 @@ /* xxHash - Fast Hash algorithm -Copyright (C) 2012-2015, Yann Collet. +Copyright (C) 2012-2015, Yann Collet + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,67 +28,71 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash +- xxHash source repository : http://code.google.com/p/xxhash/ - xxHash source mirror : https://github.com/Cyan4973/xxHash - public discussion board : https://groups.google.com/forum/#!forum/lz4c */ -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +/************************************** +* Tuning parameters +***************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. + * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). + */ #if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) # define XXH_USE_UNALIGNED_ACCESS 1 #endif -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. +/* XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/* XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1. + * It will improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ #define XXH_FORCE_NATIVE_FORMAT 0 -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif -#ifdef _MSC_VER // Visual Studio +/************************************** +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif -//************************************** -// Includes & Memory related functions -//************************************** + +/************************************** +* Includes & Memory related functions +***************************************/ #include "xxhash.h" -// Modify the local functions below should you wish to use some other memory routines -// for malloc(), free() +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ #include static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } -// for memcpy() +/* for memcpy() */ #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { @@ -95,10 +100,10 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) } -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************** +* Basic Types +***************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -144,12 +149,12 @@ typedef struct _U64_S #define A64(x) (((U64_S *)(x))->v) -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** +/***************************************** +* Compiler-specific Functions and Macros +******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ #if defined(_MSC_VER) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) @@ -158,21 +163,21 @@ typedef struct _U64_S # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif -#if defined(_MSC_VER) // Visual Studio +#if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap32 _byteswap_ulong # define XXH_swap64 _byteswap_uint64 #elif GCC_VERSION >= 403 # define XXH_swap32 __builtin_bswap32 # define XXH_swap64 __builtin_bswap64 #else -static inline U32 XXH_swap32 (U32 x) +static U32 XXH_swap32 (U32 x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } -static inline U64 XXH_swap64 (U64 x) +static U64 XXH_swap64 (U64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | @@ -186,9 +191,9 @@ static inline U64 XXH_swap64 (U64 x) #endif -//************************************** -// Constants -//************************************** +/************************************** +* Constants +***************************************/ #define PRIME32_1 2654435761U #define PRIME32_2 2246822519U #define PRIME32_3 3266489917U @@ -201,25 +206,26 @@ static inline U64 XXH_swap64 (U64 x) #define PRIME64_4 9650029242287828579ULL #define PRIME64_5 2870177450012600261ULL -//************************************** -// Architecture Macros -//************************************** + +/*************************************** +* Architecture Macros +****************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch +#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ static const int one = 1; # define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) #endif -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations +/************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ -//**************************** -// Memory reads -//**************************** +/**************************** +* Memory reads +*****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) @@ -249,9 +255,9 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) } -//**************************** -// Simple Hash Functions -//**************************** +/**************************** +* Simple Hash Functions +*****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; @@ -332,7 +338,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH unsigned int XXH32 (const void* input, size_t len, unsigned seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, input, len); @@ -341,7 +347,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 3) == 0) /* Input is aligned, let's leverage the speed advantage */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -472,7 +478,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, input, len); @@ -481,7 +487,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -529,7 +535,7 @@ typedef struct XXH32_state_t* XXH32_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) @@ -540,7 +546,7 @@ XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) @@ -591,14 +597,14 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 16) // fill in tmp buffer + if (state->memsize + len < 16) /* fill in tmp buffer */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); { @@ -745,14 +751,14 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 32) // fill in tmp buffer + if (state->memsize + len < 32) /* fill in tmp buffer */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); { diff --git a/xxhsum.c b/xxhsum.c index 3cf524c2..783c7a5c 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -38,11 +38,11 @@ You can contact the author at : /************************************** * Includes *************************************/ -#include // malloc -#include // fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno -#include // strcmp -#include // stat64 -#include // stat64 +#include /* malloc */ +#include /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */ +#include /* strcmp */ +#include /* stat64 */ +#include /* stat64 */ #include "xxhash.h" From cb1963919959fdb43dd0fd96cad84901c01beeb8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 17 Dec 2014 12:35:18 +0100 Subject: [PATCH 04/13] Fixed : minor -pedantic warnings --- xxhash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xxhash.c b/xxhash.c index a3ee31c0..093564cf 100644 --- a/xxhash.c +++ b/xxhash.c @@ -542,7 +542,7 @@ XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; -}; +} XXH64_state_t* XXH64_createState(void) { @@ -553,7 +553,7 @@ XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; -}; +} /*** Hash feed ***/ From a3f950c24c5e9ebdbb582797e53ada05301d7653 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 1 Jan 2015 17:10:04 +0100 Subject: [PATCH 05/13] updated some comments --- xxhsum.c | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/xxhsum.c b/xxhsum.c index 783c7a5c..9f5e7156 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -1,5 +1,6 @@ /* -bench.c - Demo program to benchmark open-source algorithm +xxhsum - Command line interface for xxh algorithms + Copyright (C) Yann Collet 2012-2015 This program is free software; you can redistribute it and/or modify @@ -117,33 +118,32 @@ You can contact the author at : static const char stdinName[] = "-"; -//************************************** -// Display macros -//************************************** +/************************************** + * Display macros + *************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); static unsigned g_displayLevel = 1; -//************************************** -// Unit variables -//************************************** +/************************************** + * Local variables + *************************************/ static int g_nbIterations = NBLOOPS; -static int g_fn_selection = 1; // required within main() & usage() - +static int g_fn_selection = 1; /* required within main() & usage() */ -//********************************************************* -// Benchmark Functions -//********************************************************* +/************************************** + * Benchmark Functions + *************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover + /* Based on Legacy ftime() + * Rolls over every ~ 12.1 days (0x100000/24/60/60) + * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); @@ -155,8 +155,8 @@ static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void) { - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover + /* Based on newer gettimeofday() + * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); @@ -207,7 +207,7 @@ static U64 BMK_GetFileSize(char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } @@ -216,12 +216,9 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) { int fileIdx=0; U32 hashResult=0; - U64 totals = 0; double totalc = 0.; - - // Loop for each file while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; @@ -257,9 +254,9 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) fclose(inFile); return 12; } - alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries + alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes boundaries */ - // Fill input buffer + /* Fill input buffer */ DISPLAY("\rLoading %s... \n", inFileName); readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); @@ -272,7 +269,7 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) } - // Bench XXH32 + /* XXH32 bench */ { int interationNb; double fastestC = 100000000.; From c346bb977357506b23cf40cd0ce2f2f09e679451 Mon Sep 17 00:00:00 2001 From: Jakukyo Friel Date: Wed, 7 Jan 2015 22:49:06 +0800 Subject: [PATCH 06/13] README: typo. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 06f63764..71adf884 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. -A new version, XXH64, has been created thanks to Mathias Westerdahl contribution, which offers superior speed and dispersion for 64-bits systems. Note however that 32-bits applications will still run faster using the 32-bits version. +A new version, XXH64, has been created thanks to Mathias Westerdahl's contribution, which offers superior speed and dispersion for 64-bits systems. Note however that 32-bits applications will still run faster using the 32-bits version. SMHasher speed test, compiled using GCC 4.8.2, a Linux Mint 64-bits. The reference system uses a Core i5-3340M @2.7GHz From 57fda00476d4586bae05f168201afdb933551d3b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 May 2015 22:56:53 +0100 Subject: [PATCH 07/13] sync with lz4 version --- Makefile | 12 ++--- README.md | 77 ++++++++++++++------------------ xxhash.c | 130 +++++++++++++++++++++++------------------------------- xxhash.h | 22 ++++----- xxhsum.c | 81 +++++++++++++++++----------------- 5 files changed, 145 insertions(+), 177 deletions(-) diff --git a/Makefile b/Makefile index 94cf4a93..b5071953 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # ################################################################ # xxHash Makefile -# Copyright (C) Yann Collet 2012-2014 +# Copyright (C) Yann Collet 2012-2015 +# # GPL v2 License # # This program is free software; you can redistribute it and/or modify @@ -20,10 +21,9 @@ # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ -# xxHash.exe : benchmark program, to demonstrate xxHash speed +# xxhsum : provides 32/64 bits hash of a file or pipe # ################################################################ -CC := $(CC) CFLAGS ?= -O3 CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes @@ -53,9 +53,9 @@ test: $(TEST_TARGETS) test: xxhsum ./xxhsum < xxhash.c ./xxhsum -b xxhash.c - valgrind --leak-check=yes ./xxhsum -bi1 xxhash.c - valgrind --leak-check=yes ./xxhsum -H0 xxhash.c - valgrind --leak-check=yes ./xxhsum -H1 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c test-all: test xxhsum32 ./xxhsum32 -b xxhash.c diff --git a/README.md b/README.md index 71adf884..406aa4ff 100644 --- a/README.md +++ b/README.md @@ -2,64 +2,54 @@ xxHash - Extremely fast hash algorithm ====================================== xxHash is an Extremely fast Hash algorithm, running at RAM speed limits. -It successfully passes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) Test suite evaluating Hash quality. +It successfully completes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) test suite +which evaluates collision, dispersion and randomness qualities of hash functions. |Branch |Status | |------------|---------| |master | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=master)](https://travis-ci.org/Cyan4973/xxHash?branch=master) | |dev | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=dev)](https://travis-ci.org/Cyan4973/xxHash?branch=dev) | +> **Branch Policy:** + +> - The "master" branch is considered stable, at all times. +> - The "dev" branch is the one where all contributions must be merged + before being promoted to master. +> + If you plan to propose a patch, please commit into the "dev" branch, + or its own feature branch. + Direct commit to "master" are not permitted. + Benchmarks ------------------------- -The benchmark uses SMHasher speed test, compiled with Visual on a Windows Seven 32 bits system. +The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bits box. The reference system uses a Core 2 Duo @3GHz - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameSpeedQ.ScoreAuthor
xxHash5.4 GB/s10Y.C.
MumurHash 3a2.7 GB/s10Austin Appleby
SBox1.4 GB/s9Bret Mulvey
Lookup31.2 GB/s9Bob Jenkins
CityHash641.05 GB/s10Pike & Alakuijala
FNV0.55 GB/s5Fowler, Noll, Vo
CRC320.43 GB/s9
SipHash0.34 GB/s10Jean-Philippe Aumasson
MD5-320.33 GB/s10Ronald L. Rivest
SHA1-320.28 GB/s10
+ +| Name | Speed | Quality | Author | +|---------------|----------|:-------:|------------------| +| xxHash | 5.4 GB/s | 10 | Y.C. | +| MurmurHash 3a | 2.7 GB/s | 10 | Austin Appleby | +| SBox | 1.4 GB/s | 9 | Bret Mulvey | +| Lookup3 | 1.2 GB/s | 9 | Bob Jenkins | +| CityHash64 | 1.05 GB/s| 10 | Pike & Alakuijala| +| FNV | 0.55 GB/s| 5 | Fowler, Noll, Vo | +| CRC32 | 0.43 GB/s| 9 | | +| MD5-32 | 0.33 GB/s| 10 | Ronald L.Rivest | +| SHA1-32 | 0.28 GB/s| 10 | | Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. +Algorithms with a score < 5 are not listed on this table. -A new version, XXH64, has been created thanks to Mathias Westerdahl's contribution, which offers superior speed and dispersion for 64-bits systems. Note however that 32-bits applications will still run faster using the 32-bits version. +A new version, XXH64, has been created thanks to Mathias Westerdahl's contribution, +which offers superior speed and dispersion for 64-bits systems. +Note however that 32-bits applications will still run faster using the 32-bits version. -SMHasher speed test, compiled using GCC 4.8.2, a Linux Mint 64-bits. +SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bits. The reference system uses a Core i5-3340M @2.7GHz | Version | Speed on 64-bits | Speed on 32-bits | @@ -67,8 +57,7 @@ The reference system uses a Core i5-3340M @2.7GHz | XXH64 | 13.8 GB/s | 1.9 GB/s | | XXH32 | 6.8 GB/s | 6.0 GB/s | - -This is an official mirror of xxHash project, [hosted on Google Code](http://code.google.com/p/xxhash/). -The intention is to offer github's capabilities to xxhash users, such as cloning, branch, pull requests or source download. - -The "master" branch will reflect, the status of xxhash at its official homepage. The "dev" branch is the one where all contributions will be merged. If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted. Feature branches will also exist, typically to introduce new requirements, and be temporarily available for testing before merge into "dev" branch. +Beyond the C reference version, +xxHash is also available on many programming languages, +thanks to great contributors. +They are [listed here](https://code.google.com/p/xxhash/). \ No newline at end of file diff --git a/xxhash.c b/xxhash.c index 093564cf..c88611f5 100644 --- a/xxhash.c +++ b/xxhash.c @@ -28,15 +28,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- xxHash source mirror : https://github.com/Cyan4973/xxHash -- public discussion board : https://groups.google.com/forum/#!forum/lz4c +- xxHash source repository : https://github.com/Cyan4973/xxHash */ /************************************** * Tuning parameters -***************************************/ +**************************************/ /* Unaligned memory access is automatically enabled for "common" CPU, such as x86. * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. @@ -94,10 +92,7 @@ static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } /* for memcpy() */ #include -static void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } /************************************** @@ -105,51 +100,36 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) ***************************************/ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; -#else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; #else -# define _PACKED + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; #endif -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S +static U32 XXH_read32(const void* memPtr) { - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -/***************************************** +/****************************************** * Compiler-specific Functions and Macros ******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) @@ -191,25 +171,9 @@ static U64 XXH_swap64 (U64 x) #endif -/************************************** -* Constants -***************************************/ -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - - /*************************************** * Architecture Macros -****************************************/ +***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; #ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ static const int one = 1; @@ -217,13 +181,7 @@ static const int one = 1; #endif -/************************************** -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ - - -/**************************** +/***************************** * Memory reads *****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; @@ -231,7 +189,7 @@ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); else return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); } @@ -244,7 +202,7 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); else return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); } @@ -255,7 +213,29 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) } -/**************************** +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + + +/***************************** * Simple Hash Functions *****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) @@ -335,7 +315,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned int XXH32 (const void* input, size_t len, unsigned seed) +unsigned XXH32 (const void* input, size_t len, unsigned seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ @@ -347,7 +327,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) /* Input is aligned, let's leverage the speed advantage */ + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -504,7 +484,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed } /**************************************************** - * Advanced Hash Functions +* Advanced Hash Functions ****************************************************/ /*** Allocation ***/ diff --git a/xxhash.h b/xxhash.h index be0acced..0df5ac1b 100644 --- a/xxhash.h +++ b/xxhash.h @@ -29,9 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash - - xxHash source mirror : https://github.com/Cyan4973/xxHash - - public discussion board : https://groups.google.com/forum/#!forum/lz4c + - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* Notice extracted from xxHash homepage : @@ -58,6 +56,12 @@ SHA1-32 0.28 GB/s 10 Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s */ #pragma once @@ -68,20 +72,15 @@ extern "C" { /***************************** - Includes +* Definitions *****************************/ #include /* size_t */ - - -/***************************** - Type -*****************************/ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /***************************** - Simple Hash Functions +* Simple Hash Functions *****************************/ unsigned int XXH32 (const void* input, size_t length, unsigned seed); @@ -96,12 +95,13 @@ XXH32() : Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Faster on 64-bits systems. Slower on 32-bits systems. */ /***************************** - Advanced Hash Functions +* Advanced Hash Functions *****************************/ typedef struct { long long ll[ 6]; } XXH32_state_t; typedef struct { long long ll[11]; } XXH64_state_t; diff --git a/xxhsum.c b/xxhsum.c index 9f5e7156..d65c351c 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -1,8 +1,9 @@ /* -xxhsum - Command line interface for xxh algorithms - +xxhsum - Command line interface for xxhash algorithms Copyright (C) Yann Collet 2012-2015 +GPL v2 License + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or @@ -18,14 +19,12 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash -- xxHash source mirror : https://github.com/Cyan4973/xxHash -- public discussion board : https://groups.google.com/forum/#!forum/lz4c +- xxHash source repository : https://github.com/Cyan4973/xxHash */ -/************************************** - * Compiler Options - *************************************/ +/************************************* +* Compiler Options +*************************************/ /* MS Visual */ #if defined(_MSC_VER) || defined(_WIN32) # define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ @@ -36,9 +35,9 @@ You can contact the author at : #define _LARGEFILE64_SOURCE -/************************************** - * Includes - *************************************/ +/************************************* +* Includes +*************************************/ #include /* malloc */ #include /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */ #include /* strcmp */ @@ -48,9 +47,9 @@ You can contact the author at : #include "xxhash.h" -/************************************** - * OS-Specific Includes - *************************************/ +/************************************* +* OS-Specific Includes +*************************************/ // Use ftime() if gettimeofday() is not available on your target #if defined(BMK_LEGACY_TIMER) # include // timeb, ftime @@ -77,10 +76,10 @@ You can contact the author at : #endif -/************************************** - * Basic Types - *************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************* +* Basic Types +*************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -118,25 +117,25 @@ You can contact the author at : static const char stdinName[] = "-"; -/************************************** - * Display macros - *************************************/ +/************************************* +* Display macros +*************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); static unsigned g_displayLevel = 1; -/************************************** - * Local variables - *************************************/ +/************************************* +* Local variables +*************************************/ static int g_nbIterations = NBLOOPS; static int g_fn_selection = 1; /* required within main() & usage() */ -/************************************** - * Benchmark Functions - *************************************/ +/************************************* +* Benchmark Functions +*************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) @@ -274,7 +273,7 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) int interationNb; double fastestC = 100000000.; - DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("\r%79s\r", ""); /* Clean display line */ for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) { int nbHashes = 0; @@ -282,7 +281,7 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); - // Hash loop + /* Timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); @@ -305,19 +304,19 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) totalc += fastestC; } - // Bench Unaligned XXH32 + /* Bench XXH32 on Unaligned input */ { int interationNb; double fastestC = 100000000.; - DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("\r%79s\r", ""); /* Clean display line */ for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++) { int nbHashes = 0; int milliTime; DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); - // Hash loop + /* timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); @@ -334,16 +333,16 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); + DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); } - // Bench XXH64 + /* Bench XXH64 */ { int interationNb; double fastestC = 100000000.; unsigned long long h64 = 0; - DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("\r%79s\r", ""); /* Clean display line */ for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) { int nbHashes = 0; @@ -351,7 +350,7 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); - // Hash loop + /* Timing loop */ milliTime = BMK_GetMilliStart(); while(BMK_GetMilliStart() == milliTime); milliTime = BMK_GetMilliStart(); @@ -486,7 +485,7 @@ static void BMK_sanityCheck(void) BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); - DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("\r%79s\r", ""); /* Clean display line */ DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } @@ -590,9 +589,9 @@ static int BMK_hash(const char* fileName, U32 hashNb) } -//********************************************************* -// Main -//********************************************************* +/********************************************************* +* Main +*********************************************************/ static int usage(const char* exename) { @@ -624,7 +623,7 @@ int main(int argc, char** argv) const char* exename = argv[0]; U32 benchmarkMode = 0; - // xxh32sum default to 32 bits checksum + /* xxh32sum default to 32 bits checksum */ if (strstr(exename, "xxh32sum")!=NULL) g_fn_selection=0; for(i=1; i Date: Tue, 5 May 2015 01:01:10 +0100 Subject: [PATCH 08/13] added more tests --- .travis.yml | 5 ++++- Makefile | 47 +++++++++++++++++++++++++++++++++++++---------- xxhsum.c | 2 +- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1a45fc2..f6f0597c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,10 @@ language: c compiler: gcc script: make test-all before_install: - - sudo apt-get update -qq + - sudo apt-get update -qq + - sudo apt-get install -qq gcc-arm-linux-gnueabi + - sudo apt-get install -qq clang + - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind diff --git a/Makefile b/Makefile index b5071953..5e808801 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,12 @@ # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ -# xxhsum : provides 32/64 bits hash of a file or pipe +# xxhsum : provides 32/64 bits hash of a file or piped data # ################################################################ CFLAGS ?= -O3 -CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic +FLAGS := -I. $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems @@ -41,24 +42,50 @@ default: xxhsum all: xxhsum xxhsum32 xxhsum: xxhash.c xxhsum.c - $(CC) $(CFLAGS) $^ -o $@$(EXT) + $(CC) $(FLAGS) $^ -o $@$(EXT) ln -sf $@ xxh32sum ln -sf $@ xxh64sum xxhsum32: xxhash.c xxhsum.c - $(CC) -m32 $(CFLAGS) $^ -o $@$(EXT) + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -test: $(TEST_TARGETS) - -test: xxhsum +test: clean xxhsum ./xxhsum < xxhash.c - ./xxhsum -b xxhash.c + ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c -test-all: test xxhsum32 - ./xxhsum32 -b xxhash.c +test32: clean xxhsum32 + @echo ---- test 32-bits ---- + ./xxhsum32 -bi1 xxhash.c + +armtest: + @echo ---- test ARM compilation ---- + $(MAKE) clean + $(MAKE) xxhsum CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + +clangtest: + @echo ---- test clang compilation ---- + $(MAKE) clean + $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" + +gpptest: + @echo ---- test g++ compilation ---- + $(MAKE) clean + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +sanitize: + @echo ---- check undefined behavior - sanitize ---- + $(MAKE) clean + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" + +staticAnalyze: + @echo ---- static analyzer - scan-build ---- + $(MAKE) clean + scan-build --status-bugs -v $(MAKE) all MOREFLAGS=-g + +test-all: test test32 armtest clangtest gpptest sanitize staticAnalyze clean: @rm -f core *.o xxhsum$(EXT) xxhsum32$(EXT) xxh32sum xxh64sum diff --git a/xxhsum.c b/xxhsum.c index d65c351c..ab51c157 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -491,7 +491,7 @@ static void BMK_sanityCheck(void) static void BMK_display_BigEndian(const void* ptr, size_t length) { - const BYTE* p = ptr; + const BYTE* p = (const BYTE*)ptr; while (length--) DISPLAYRESULT("%02x", *p++); } From 3b63fc28cac451f0179145467e6410d17a926bf7 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 May 2015 11:25:28 +0100 Subject: [PATCH 09/13] Added -Wcast-qual compilation flag, as suggested by Isaac Turner --- Makefile | 2 +- xxhash.c | 14 +++++++------- xxhsum.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 5e808801..a8f0ef8c 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ # ################################################################ CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -Wcast-qual FLAGS := -I. $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS) diff --git a/xxhash.c b/xxhash.c index c88611f5..e6fb8f14 100644 --- a/xxhash.c +++ b/xxhash.c @@ -177,7 +177,7 @@ static U64 XXH_swap64 (U64 x) typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; #ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) #endif @@ -191,7 +191,7 @@ FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_a if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); else - return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) @@ -204,7 +204,7 @@ FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_a if (align==XXH_unaligned) return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); else - return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) @@ -668,9 +668,9 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { - XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->total_len >= 16) @@ -822,9 +822,9 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { - XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) diff --git a/xxhsum.c b/xxhsum.c index ab51c157..38388bbe 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -619,7 +619,7 @@ static int badusage(const char* exename) int main(int argc, char** argv) { int i, filenamesStart=0; - const char* input_filename = (char*)stdinName; + const char* input_filename = (const char*)stdinName; const char* exename = argv[0]; U32 benchmarkMode = 0; From 76c90f1d0f0ba096b166efbbdde25b08a47f5009 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 May 2015 13:30:27 +0100 Subject: [PATCH 10/13] Added : internal benchmark mode --- Makefile | 5 +- xxhsum.c | 272 ++++++++++++++++++++++++++++++------------------------- 2 files changed, 153 insertions(+), 124 deletions(-) diff --git a/Makefile b/Makefile index a8f0ef8c..b233bf6b 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,11 @@ # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ -# xxhsum : provides 32/64 bits hash of a file or piped data +# xxhsum : provides 32/64 bits hash of a file, or piped data # ################################################################ CFLAGS ?= -O3 -CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes -pedantic -Wcast-qual +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic FLAGS := -I. $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS) @@ -51,6 +51,7 @@ xxhsum32: xxhash.c xxhsum.c test: clean xxhsum ./xxhsum < xxhash.c + ./xxhsum -bi1 ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c diff --git a/xxhsum.c b/xxhsum.c index 38388bbe..43ed4396 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -211,12 +211,123 @@ static U64 BMK_GetFileSize(char* infilename) } +/* Note : buffer is supposed malloc'ed, hence aligned */ +static void BMK_benchMem(const void* buffer, size_t bufferSize) +{ + static const int nbh_perloop = 100; + + /* XXH32 bench */ + { + int iterationNb; + double fastestH = 100000000.; + U32 hashResult; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH32", (int)bufferSize); + + /* Timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000., hashResult); + } + + /* Bench XXH32 on Unaligned input */ + { + int iterationNb; + double fastestH = 100000000.; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + for (iterationNb = 1; (iterationNb <= g_nbIterations) && ((bufferSize>1)); iterationNb++) + { + int nbHashes = 0; + int milliTime; + const char* charPtr = (const char*)buffer; + + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "(unaligned)", (int)(bufferSize-1)); + /* timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + } + + /* Bench XXH64 */ + { + int iterationNb; + double fastestH = 100000000.; + unsigned long long h64 = 0; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH64", (int)bufferSize); + + /* Timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + } + { + int i; + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + for (i=7; i>=0; i--) + DISPLAY("%02X", (U32)(h64 >> (i*8)) & 0xFF); + DISPLAY("\n"); + } + } +} + + static int BMK_benchFile(char** fileNamesTable, int nbFiles) { int fileIdx=0; - U32 hashResult=0; - U64 totals = 0; - double totalc = 0.; while (fileIdx\r", interationNb, "XXH32", (int)benchedSize); - - /* Timing loop */ - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - hashResult = XXH32(alignedBuffer, benchedSize, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); - - totals += benchedSize; - totalc += fastestC; - } - - /* Bench XXH32 on Unaligned input */ - { - int interationNb; - double fastestC = 100000000.; - - DISPLAY("\r%79s\r", ""); /* Clean display line */ - for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++) - { - int nbHashes = 0; - int milliTime; - - DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); - /* timing loop */ - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - hashResult = XXH32(alignedBuffer+1, benchedSize-1, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); - } - - /* Bench XXH64 */ - { - int interationNb; - double fastestC = 100000000.; - unsigned long long h64 = 0; + free(buffer); + } - DISPLAY("\r%79s\r", ""); /* Clean display line */ - for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) - { - int nbHashes = 0; - int milliTime; + return 0; +} - DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); - /* Timing loop */ - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - h64 = XXH64(alignedBuffer, benchedSize, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); - totals += benchedSize; - totalc += fastestC; - } +static int BMK_benchInternal(void) +{ + static const size_t benchedSize = 100 KB; + void* buffer; - free(buffer); + buffer = malloc(benchedSize); + if(!buffer) + { + DISPLAY("\nError: not enough memory!\n"); + return 12; } - if (nbFiles > 1) - printf("%-16.16s :%11llu -> %7.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (double)totals/totalc/1000.); + /* bench */ + DISPLAY("\rSample of %u KB... \n", (U32)(benchedSize >> 10)); + BMK_benchMem(buffer, benchedSize); + + free(buffer); return 0; } - static void BMK_checkResult(U32 r1, U32 r2) { static int nbTests = 1; @@ -489,12 +516,14 @@ static void BMK_sanityCheck(void) DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } + static void BMK_display_BigEndian(const void* ptr, size_t length) { const BYTE* p = (const BYTE*)ptr; while (length--) DISPLAYRESULT("%02x", *p++); } + static int BMK_hash(const char* fileName, U32 hashNb) { FILE* inFile; @@ -503,7 +532,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) char* buffer; XXH64_state_t state; - // Check file existence + /* Check file existence */ if (fileName == stdinName) { inFile = stdin; @@ -517,7 +546,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) return 11; } - // Memory allocation & restrictions + /* Memory allocation & restrictions */ buffer = (char*)malloc(blockSize); if(!buffer) { @@ -526,7 +555,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) return 12; } - // Init + /* Init */ switch(hashNb) { case 0: @@ -543,7 +572,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) } - // Load file & update hash + /* Load file & update hash */ DISPLAY("\rLoading %s... \r", fileName); readSize = 1; while (readSize) @@ -564,7 +593,7 @@ static int BMK_hash(const char* fileName, U32 hashNb) fclose(inFile); free(buffer); - // display Hash + /* display Hash */ switch(hashNb) { case 0: @@ -630,7 +659,7 @@ int main(int argc, char** argv) { char* argument = argv[i]; - if(!argument) continue; // Protection if argument empty + if(!argument) continue; /* Protection, if argument empty */ if (*argument!='-') { @@ -639,9 +668,8 @@ int main(int argc, char** argv) continue; } - // Select command - // note : *argument=='-' - argument++; + /* command selection */ + argument++; /* note : *argument=='-' */ while (*argument!=0) { @@ -657,7 +685,7 @@ int main(int argc, char** argv) argument+=2; break; - // Trigger benchmark mode + /* Trigger benchmark mode */ case 'b': argument++; benchmarkMode=1; @@ -675,18 +703,18 @@ int main(int argc, char** argv) } } - // Check if input is defined as console; trigger an error in this case - if ((input_filename == stdinName) && IS_CONSOLE(stdin) ) return badusage(exename); - - // Check results are good + /* Check benchmark mode */ if (benchmarkMode) { - if (filenamesStart==0) return badusage(exename); DISPLAY( WELCOME_MESSAGE ); BMK_sanityCheck(); + if (filenamesStart==0) return BMK_benchInternal(); return BMK_benchFile(argv+filenamesStart, argc-filenamesStart); } + /* Check if input is defined as console; trigger an error in this case */ + if ((input_filename == stdinName) && IS_CONSOLE(stdin) ) return badusage(exename); + if(g_fn_selection < 0 || g_fn_selection > 1) return badusage(exename); return BMK_hash(input_filename, g_fn_selection); From 962fd0ab4d12c51ce3fdb0bde5c672d56b19d765 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 May 2015 15:27:27 +0100 Subject: [PATCH 11/13] restored big-endian display --- xxhsum.c | 87 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/xxhsum.c b/xxhsum.c index 43ed4396..72cca5d3 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -50,23 +50,23 @@ You can contact the author at : /************************************* * OS-Specific Includes *************************************/ -// Use ftime() if gettimeofday() is not available on your target +/* Use ftime() if gettimeofday() is not available on your target */ #if defined(BMK_LEGACY_TIMER) -# include // timeb, ftime +# include /* timeb, ftime */ #else -# include // gettimeofday +# include /* gettimeofday */ #endif #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include // _O_BINARY -# include // _setmode, _isatty +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ # ifdef __MINGW32__ - int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into + int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -# include // isatty, STDIN_FILENO +# include /* isatty, STDIN_FILENO */ # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(STDIN_FILENO) #endif @@ -94,22 +94,27 @@ You can contact the author at : typedef unsigned long long U64; #endif +static unsigned BMK_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + /************************************** - * Constants - *************************************/ +* Constants +**************************************/ #define PROGRAM_NAME exename #define PROGRAM_VERSION "" #define COMPILED __DATE__ -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED +static const char author[] = "Yann Collet"; +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, author, COMPILED -#define NBLOOPS 3 // Default number of benchmark iterations -#define TIMELOOP 2500 // Minimum timing per iteration -#define PRIME 2654435761U +#define NBLOOPS 3 /* Default number of benchmark iterations */ +#define TIMELOOP 2500 /* Minimum timing per iteration */ -#define KB *(1<<10) -#define MB *(1<<20) +#define KB *( 1<<10) +#define MB *( 1<<20) #define GB *(1U<<30) #define MAX_MEM (2 GB - 64 MB) @@ -315,10 +320,8 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); } { - int i; DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); - for (i=7; i>=0; i--) - DISPLAY("%02X", (U32)(h64 >> (i*8)) & 0xFF); + DISPLAY("%08X%08X", (U32)(h64 >> 32), (U32)h64); DISPLAY("\n"); } } @@ -432,7 +435,7 @@ static void BMK_checkResult64(U64 r1, U64 r2) if (r1!=r2) { DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); - DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2); + DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2); exit(1); } nbTests++; @@ -460,22 +463,22 @@ static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) } -static void BMK_testSequence(void* sentence, int len, U32 seed, U32 Nresult) +static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult) { U32 Dresult; XXH32_state_t state; - int index; + size_t index; - Dresult = XXH32(sentence, len, seed); + Dresult = XXH32(sequence, len, seed); BMK_checkResult(Dresult, Nresult); XXH32_reset(&state, seed); - XXH32_update(&state, sentence, len); + XXH32_update(&state, sequence, len); Dresult = XXH32_digest(&state); BMK_checkResult(Dresult, Nresult); XXH32_reset(&state, seed); - for (index=0; index>24); - prime *= prime; + sanityBuffer[i] = (BYTE)(byteGen>>24); + byteGen *= byteGen; } BMK_testSequence(NULL, 0, 0, 0x02CC5D05); - BMK_testSequence(NULL, 0, PRIME, 0x36B78AE7); + BMK_testSequence(NULL, 0, prime, 0x36B78AE7); BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); - BMK_testSequence(sanityBuffer, 1, PRIME, 0xD5845D64); + BMK_testSequence(sanityBuffer, 1, prime, 0xD5845D64); BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4); - BMK_testSequence(sanityBuffer, 14, PRIME, 0x4481951D); + BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); - BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2); + BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2); BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL); - BMK_testSequence64(NULL , 0, PRIME, 0xAC75FDA2929B17EFULL); + BMK_testSequence64(NULL , 0, prime, 0xAC75FDA2929B17EFULL); BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); - BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL); + BMK_testSequence64(sanityBuffer, 1, prime, 0x739840CB819FA723ULL); BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); - BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL); + BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); - BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL); DISPLAY("\r%79s\r", ""); /* Clean display line */ DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); @@ -520,7 +525,9 @@ static void BMK_sanityCheck(void) static void BMK_display_BigEndian(const void* ptr, size_t length) { const BYTE* p = (const BYTE*)ptr; - while (length--) DISPLAYRESULT("%02x", *p++); + size_t index = BMK_isLittleEndian() ? length-1 : 0; + int incr = BMK_isLittleEndian() ? -1 : 1; + while (index Date: Sun, 28 Jun 2015 03:20:23 -0800 Subject: [PATCH 12/13] Added : XXH_NAMESPACE, for namespace emulation in C --- Makefile | 16 ++++++---------- xxhash.h | 33 +++++++++++++++++++++++++++++++++ xxhsum.c | 28 ++++++++++++++++------------ 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index b233bf6b..11db9d7a 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ else EXT = endif +.PHONY: clean all default: xxhsum @@ -61,29 +62,24 @@ test32: clean xxhsum32 @echo ---- test 32-bits ---- ./xxhsum32 -bi1 xxhash.c -armtest: +armtest: clean @echo ---- test ARM compilation ---- - $(MAKE) clean $(MAKE) xxhsum CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" -clangtest: +clangtest: clean @echo ---- test clang compilation ---- - $(MAKE) clean $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" -gpptest: +gpptest: clean @echo ---- test g++ compilation ---- - $(MAKE) clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" -sanitize: +sanitize: clean @echo ---- check undefined behavior - sanitize ---- - $(MAKE) clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" -staticAnalyze: +staticAnalyze: clean @echo ---- static analyzer - scan-build ---- - $(MAKE) clean scan-build --status-bugs -v $(MAKE) all MOREFLAGS=-g test-all: test test32 armtest clangtest gpptest sanitize staticAnalyze diff --git a/xxhash.h b/xxhash.h index 0df5ac1b..c60aa615 100644 --- a/xxhash.h +++ b/xxhash.h @@ -78,6 +78,39 @@ extern "C" { typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; +/***************************** +* Namespace Emulation +*****************************/ +/* Motivations : + +If you need to include xxHash into your library, +but wish to avoid xxHash symbols to be present on your library interface +in an effort to avoid potential name collision if another library also includes xxHash, + +you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash +with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). + +Note that no change is required within the calling program : +it can still call xxHash functions using their regular name. +They will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif + /***************************** * Simple Hash Functions diff --git a/xxhsum.c b/xxhsum.c index 72cca5d3..f9cd9b83 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -179,25 +179,28 @@ static int BMK_GetMilliSpan( int nTimeStart ) } -static size_t BMK_findMaxMem(U64 requestedMem) +static size_t BMK_findMaxMem(U64 requiredMem) { - size_t step = (64 MB); - size_t allocatedMemory; + size_t step = 64 MB; BYTE* testmem=NULL; - requestedMem += 3*step; - requestedMem -= (size_t)requestedMem & (step-1); - if (requestedMem > MAX_MEM) requestedMem = MAX_MEM; - allocatedMemory = (size_t)requestedMem; + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; while (!testmem) { - allocatedMemory -= step; - testmem = (BYTE*) malloc((size_t)allocatedMemory); + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = (BYTE*) malloc ((size_t)requiredMem); } free (testmem); - return (size_t) (allocatedMemory - step); + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; } @@ -225,9 +228,10 @@ static void BMK_benchMem(const void* buffer, size_t bufferSize) { int iterationNb; double fastestH = 100000000.; - U32 hashResult; + U32 hashResult = 0; DISPLAY("\r%79s\r", ""); /* Clean display line */ + if (g_nbIterations<1) g_nbIterations=1; for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) { int nbHashes = 0; @@ -345,7 +349,7 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) /* Check file existence */ inFileName = fileNamesTable[fileIdx++]; inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) + if ((inFile==NULL) || (inFileName==NULL)) { DISPLAY( "Pb opening %s\n", inFileName); return 11; From fcc80928a31c954ca348b04b18c47ea103b1103a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Jun 2015 03:40:12 -0800 Subject: [PATCH 13/13] travis : force targets to be rebuilt --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f6f0597c..4adeb390 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: c compiler: gcc -script: make test-all +script: make -B test-all before_install: - sudo apt-get update -qq - sudo apt-get install -qq gcc-arm-linux-gnueabi