diff --git a/.travis.yml b/.travis.yml index f1a45fc2..4adeb390 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,11 @@ language: c compiler: gcc -script: make test-all +script: make -B test-all before_install: - - sudo apt-get update -qq + - sudo apt-get update -qq + - sudo apt-get install -qq gcc-arm-linux-gnueabi + - sudo apt-get install -qq clang + - sudo apt-get install -qq g++-multilib - sudo apt-get install -qq gcc-multilib - sudo apt-get install -qq valgrind diff --git a/Makefile b/Makefile index 94cf4a93..11db9d7a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # ################################################################ # xxHash Makefile -# Copyright (C) Yann Collet 2012-2014 +# Copyright (C) Yann Collet 2012-2015 +# # GPL v2 License # # This program is free software; you can redistribute it and/or modify @@ -20,12 +21,12 @@ # You can contact the author at : # - xxHash source repository : http://code.google.com/p/xxhash/ # ################################################################ -# xxHash.exe : benchmark program, to demonstrate xxHash speed +# xxhsum : provides 32/64 bits hash of a file, or piped data # ################################################################ -CC := $(CC) CFLAGS ?= -O3 -CFLAGS += -I. -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes +CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic +FLAGS := -I. $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS) # Define *.exe as extension for Windows systems @@ -35,30 +36,53 @@ else EXT = endif +.PHONY: clean all default: xxhsum all: xxhsum xxhsum32 xxhsum: xxhash.c xxhsum.c - $(CC) $(CFLAGS) $^ -o $@$(EXT) + $(CC) $(FLAGS) $^ -o $@$(EXT) ln -sf $@ xxh32sum ln -sf $@ xxh64sum xxhsum32: xxhash.c xxhsum.c - $(CC) -m32 $(CFLAGS) $^ -o $@$(EXT) - -test: $(TEST_TARGETS) + $(CC) -m32 $(FLAGS) $^ -o $@$(EXT) -test: xxhsum +test: clean xxhsum ./xxhsum < xxhash.c - ./xxhsum -b xxhash.c - valgrind --leak-check=yes ./xxhsum -bi1 xxhash.c - valgrind --leak-check=yes ./xxhsum -H0 xxhash.c - valgrind --leak-check=yes ./xxhsum -H1 xxhash.c + ./xxhsum -bi1 + ./xxhsum -bi1 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c + valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c + +test32: clean xxhsum32 + @echo ---- test 32-bits ---- + ./xxhsum32 -bi1 xxhash.c + +armtest: clean + @echo ---- test ARM compilation ---- + $(MAKE) xxhsum CC=arm-linux-gnueabi-gcc MOREFLAGS="-Werror" + +clangtest: clean + @echo ---- test clang compilation ---- + $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" + +gpptest: clean + @echo ---- test g++ compilation ---- + $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" + +sanitize: clean + @echo ---- check undefined behavior - sanitize ---- + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" + +staticAnalyze: clean + @echo ---- static analyzer - scan-build ---- + scan-build --status-bugs -v $(MAKE) all MOREFLAGS=-g -test-all: test xxhsum32 - ./xxhsum32 -b xxhash.c +test-all: test test32 armtest clangtest gpptest sanitize staticAnalyze clean: @rm -f core *.o xxhsum$(EXT) xxhsum32$(EXT) xxh32sum xxh64sum diff --git a/README.md b/README.md index 06f63764..406aa4ff 100644 --- a/README.md +++ b/README.md @@ -2,64 +2,54 @@ xxHash - Extremely fast hash algorithm ====================================== xxHash is an Extremely fast Hash algorithm, running at RAM speed limits. -It successfully passes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) Test suite evaluating Hash quality. +It successfully completes the [SMHasher](http://code.google.com/p/smhasher/wiki/SMHasher) test suite +which evaluates collision, dispersion and randomness qualities of hash functions. |Branch |Status | |------------|---------| |master | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=master)](https://travis-ci.org/Cyan4973/xxHash?branch=master) | |dev | [![Build Status](https://travis-ci.org/Cyan4973/xxHash.svg?branch=dev)](https://travis-ci.org/Cyan4973/xxHash?branch=dev) | +> **Branch Policy:** + +> - The "master" branch is considered stable, at all times. +> - The "dev" branch is the one where all contributions must be merged + before being promoted to master. +> + If you plan to propose a patch, please commit into the "dev" branch, + or its own feature branch. + Direct commit to "master" are not permitted. + Benchmarks ------------------------- -The benchmark uses SMHasher speed test, compiled with Visual on a Windows Seven 32 bits system. +The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bits box. The reference system uses a Core 2 Duo @3GHz - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameSpeedQ.ScoreAuthor
xxHash5.4 GB/s10Y.C.
MumurHash 3a2.7 GB/s10Austin Appleby
SBox1.4 GB/s9Bret Mulvey
Lookup31.2 GB/s9Bob Jenkins
CityHash641.05 GB/s10Pike & Alakuijala
FNV0.55 GB/s5Fowler, Noll, Vo
CRC320.43 GB/s9
SipHash0.34 GB/s10Jean-Philippe Aumasson
MD5-320.33 GB/s10Ronald L. Rivest
SHA1-320.28 GB/s10
+ +| Name | Speed | Quality | Author | +|---------------|----------|:-------:|------------------| +| xxHash | 5.4 GB/s | 10 | Y.C. | +| MurmurHash 3a | 2.7 GB/s | 10 | Austin Appleby | +| SBox | 1.4 GB/s | 9 | Bret Mulvey | +| Lookup3 | 1.2 GB/s | 9 | Bob Jenkins | +| CityHash64 | 1.05 GB/s| 10 | Pike & Alakuijala| +| FNV | 0.55 GB/s| 5 | Fowler, Noll, Vo | +| CRC32 | 0.43 GB/s| 9 | | +| MD5-32 | 0.33 GB/s| 10 | Ronald L.Rivest | +| SHA1-32 | 0.28 GB/s| 10 | | Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. +Algorithms with a score < 5 are not listed on this table. -A new version, XXH64, has been created thanks to Mathias Westerdahl contribution, which offers superior speed and dispersion for 64-bits systems. Note however that 32-bits applications will still run faster using the 32-bits version. +A new version, XXH64, has been created thanks to Mathias Westerdahl's contribution, +which offers superior speed and dispersion for 64-bits systems. +Note however that 32-bits applications will still run faster using the 32-bits version. -SMHasher speed test, compiled using GCC 4.8.2, a Linux Mint 64-bits. +SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bits. The reference system uses a Core i5-3340M @2.7GHz | Version | Speed on 64-bits | Speed on 32-bits | @@ -67,8 +57,7 @@ The reference system uses a Core i5-3340M @2.7GHz | XXH64 | 13.8 GB/s | 1.9 GB/s | | XXH32 | 6.8 GB/s | 6.0 GB/s | - -This is an official mirror of xxHash project, [hosted on Google Code](http://code.google.com/p/xxhash/). -The intention is to offer github's capabilities to xxhash users, such as cloning, branch, pull requests or source download. - -The "master" branch will reflect, the status of xxhash at its official homepage. The "dev" branch is the one where all contributions will be merged. If you plan to propose a patch, please commit into the "dev" branch. Direct commit to "master" are not permitted. Feature branches will also exist, typically to introduce new requirements, and be temporarily available for testing before merge into "dev" branch. +Beyond the C reference version, +xxHash is also available on many programming languages, +thanks to great contributors. +They are [listed here](https://code.google.com/p/xxhash/). \ No newline at end of file diff --git a/xxhash.c b/xxhash.c index 24a64b5f..e6fb8f14 100644 --- a/xxhash.c +++ b/xxhash.c @@ -1,6 +1,7 @@ /* xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. +Copyright (C) 2012-2015, Yann Collet + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -27,128 +28,113 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -- public discussion board : https://groups.google.com/forum/#!forum/lz4c +- xxHash source repository : https://github.com/Cyan4973/xxHash */ -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +/************************************** +* Tuning parameters +**************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. + * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). + */ #if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) # define XXH_USE_UNALIGNED_ACCESS 1 #endif -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. +/* XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/* XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1. + * It will improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ #define XXH_FORCE_NATIVE_FORMAT 0 -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif -#ifdef _MSC_VER // Visual Studio +/************************************** +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif # else -# define FORCE_INLINE static inline -# endif +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ #endif -//************************************** -// Includes & Memory related functions -//************************************** + +/************************************** +* Includes & Memory related functions +***************************************/ #include "xxhash.h" -// Modify the local functions below should you wish to use some other memory routines -// for malloc(), free() +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ #include static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } -// for memcpy() +/* for memcpy() */ #include -static void* XXH_memcpy(void* dest, const void* src, size_t size) -{ - return memcpy(dest,src,size); -} +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************** +* Basic Types +***************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include -typedef uint8_t BYTE; -typedef uint16_t U16; -typedef uint32_t U32; -typedef int32_t S32; -typedef uint64_t U64; + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; #else -typedef unsigned char BYTE; -typedef unsigned short U16; -typedef unsigned int U32; -typedef signed int S32; -typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; #endif -typedef struct _U32_S +static U32 XXH_read32(const void* memPtr) { - U32 v; -} _PACKED U32_S; -typedef struct _U64_S -{ - U64 v; -} _PACKED U64_S; + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} -#define A32(x) (((U32_S *)(x))->v) -#define A64(x) (((U64_S *)(x))->v) -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** +/****************************************** +* Compiler-specific Functions and Macros +******************************************/ #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ #if defined(_MSC_VER) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) @@ -157,21 +143,21 @@ typedef struct _U64_S # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif -#if defined(_MSC_VER) // Visual Studio +#if defined(_MSC_VER) /* Visual Studio */ # define XXH_swap32 _byteswap_ulong # define XXH_swap64 _byteswap_uint64 #elif GCC_VERSION >= 403 # define XXH_swap32 __builtin_bswap32 # define XXH_swap64 __builtin_bswap64 #else -static inline U32 XXH_swap32 (U32 x) +static U32 XXH_swap32 (U32 x) { return ((x << 24) & 0xff000000 ) | ((x << 8) & 0x00ff0000 ) | ((x >> 8) & 0x0000ff00 ) | ((x >> 24) & 0x000000ff ); } -static inline U64 XXH_swap64 (U64 x) +static U64 XXH_swap64 (U64 x) { return ((x << 56) & 0xff00000000000000ULL) | ((x << 40) & 0x00ff000000000000ULL) | @@ -185,48 +171,27 @@ static inline U64 XXH_swap64 (U64 x) #endif -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - -#define PRIME64_1 11400714785074694791ULL -#define PRIME64_2 14029467366897019727ULL -#define PRIME64_3 1609587929392839161ULL -#define PRIME64_4 9650029242287828579ULL -#define PRIME64_5 2870177450012600261ULL - -//************************************** -// Architecture Macros -//************************************** +/*************************************** +* Architecture Macros +***************************************/ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch +#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) #endif -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** +/***************************** +* Memory reads +*****************************/ typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); else - return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr); + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); } FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) @@ -237,9 +202,9 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) { if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); else - return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr); + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); } FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) @@ -248,9 +213,31 @@ FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) } -//**************************** -// Simple Hash Functions -//**************************** +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + + +/***************************** +* Simple Hash Functions +*****************************/ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; @@ -328,10 +315,10 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH } -unsigned int XXH32 (const void* input, size_t len, unsigned seed) +unsigned XXH32 (const void* input, size_t len, unsigned seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH32_state_t state; XXH32_reset(&state, seed); XXH32_update(&state, input, len); @@ -340,7 +327,7 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed) XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -471,7 +458,7 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ XXH64_state_t state; XXH64_reset(&state, seed); XXH64_update(&state, input, len); @@ -480,7 +467,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; # if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ { if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); @@ -497,7 +484,7 @@ unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed } /**************************************************** - * Advanced Hash Functions +* Advanced Hash Functions ****************************************************/ /*** Allocation ***/ @@ -528,25 +515,25 @@ typedef struct XXH32_state_t* XXH32_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; -}; +} XXH64_state_t* XXH64_createState(void) { - XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; -}; +} /*** Hash feed ***/ @@ -590,14 +577,14 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 16) // fill in tmp buffer + if (state->memsize + len < 16) /* fill in tmp buffer */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); { @@ -681,9 +668,9 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) { - XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; const BYTE * p = (const BYTE*)state->mem32; - BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->total_len >= 16) @@ -744,14 +731,14 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v state->total_len += len; - if (state->memsize + len < 32) // fill in tmp buffer + if (state->memsize + len < 32) /* fill in tmp buffer */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); state->memsize += (U32)len; return XXH_OK; } - if (state->memsize) // some data left from previous update + if (state->memsize) /* some data left from previous update */ { XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); { @@ -835,9 +822,9 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) { - XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; const BYTE * p = (const BYTE*)state->mem64; - BYTE* bEnd = (BYTE*)state->mem64 + state->memsize; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) diff --git a/xxhash.h b/xxhash.h index 55b45015..c60aa615 100644 --- a/xxhash.h +++ b/xxhash.h @@ -1,7 +1,8 @@ /* xxHash - Extremely Fast Hash algorithm Header File - Copyright (C) 2012-2014, Yann Collet. + Copyright (C) 2012-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without @@ -28,7 +29,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ + - xxHash source repository : https://github.com/Cyan4973/xxHash */ /* Notice extracted from xxHash homepage : @@ -55,6 +56,12 @@ SHA1-32 0.28 GB/s 10 Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s */ #pragma once @@ -65,20 +72,48 @@ extern "C" { /***************************** - Includes +* Definitions *****************************/ #include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /***************************** - Type +* Namespace Emulation *****************************/ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; +/* Motivations : +If you need to include xxHash into your library, +but wish to avoid xxHash symbols to be present on your library interface +in an effort to avoid potential name collision if another library also includes xxHash, + +you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash +with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). + +Note that no change is required within the calling program : +it can still call xxHash functions using their regular name. +They will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif /***************************** - Simple Hash Functions +* Simple Hash Functions *****************************/ unsigned int XXH32 (const void* input, size_t length, unsigned seed); @@ -93,12 +128,13 @@ XXH32() : Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s XXH64() : Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Faster on 64-bits systems. Slower on 32-bits systems. */ /***************************** - Advanced Hash Functions +* Advanced Hash Functions *****************************/ typedef struct { long long ll[ 6]; } XXH32_state_t; typedef struct { long long ll[11]; } XXH64_state_t; diff --git a/xxhsum.c b/xxhsum.c index e090e511..f9cd9b83 100644 --- a/xxhsum.c +++ b/xxhsum.c @@ -1,6 +1,8 @@ /* -bench.c - Demo program to benchmark open-source algorithm -Copyright (C) Yann Collet 2012-2014 +xxhsum - Command line interface for xxhash algorithms +Copyright (C) Yann Collet 2012-2015 + +GPL v2 License This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,13 +19,12 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : -- Blog homepage : http://fastcompression.blogspot.com/ -- Discussion group : https://groups.google.com/forum/?fromgroups#!forum/lz4c +- xxHash source repository : https://github.com/Cyan4973/xxHash */ -/************************************** - * Compiler Options - *************************************/ +/************************************* +* Compiler Options +*************************************/ /* MS Visual */ #if defined(_MSC_VER) || defined(_WIN32) # define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ @@ -34,38 +35,38 @@ You can contact the author at : #define _LARGEFILE64_SOURCE -/************************************** - * Includes - *************************************/ -#include // malloc -#include // fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno -#include // strcmp -#include // stat64 -#include // stat64 +/************************************* +* Includes +*************************************/ +#include /* malloc */ +#include /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */ +#include /* strcmp */ +#include /* stat64 */ +#include /* stat64 */ #include "xxhash.h" -/************************************** - * OS-Specific Includes - *************************************/ -// Use ftime() if gettimeofday() is not available on your target +/************************************* +* OS-Specific Includes +*************************************/ +/* Use ftime() if gettimeofday() is not available on your target */ #if defined(BMK_LEGACY_TIMER) -# include // timeb, ftime +# include /* timeb, ftime */ #else -# include // gettimeofday +# include /* gettimeofday */ #endif #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) -# include // _O_BINARY -# include // _setmode, _isatty +# include /* _O_BINARY */ +# include /* _setmode, _isatty */ # ifdef __MINGW32__ - int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into + int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into */ # endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -# include // isatty, STDIN_FILENO +# include /* isatty, STDIN_FILENO */ # define SET_BINARY_MODE(file) # define IS_CONSOLE(stdStream) isatty(STDIN_FILENO) #endif @@ -75,10 +76,10 @@ You can contact the author at : #endif -/************************************** - * Basic Types - *************************************/ -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +/************************************* +* Basic Types +*************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; @@ -93,22 +94,27 @@ You can contact the author at : typedef unsigned long long U64; #endif +static unsigned BMK_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + /************************************** - * Constants - *************************************/ +* Constants +**************************************/ #define PROGRAM_NAME exename #define PROGRAM_VERSION "" #define COMPILED __DATE__ -#define AUTHOR "Yann Collet" -#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED +static const char author[] = "Yann Collet"; +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, author, COMPILED -#define NBLOOPS 3 // Default number of benchmark iterations -#define TIMELOOP 2500 // Minimum timing per iteration -#define PRIME 2654435761U +#define NBLOOPS 3 /* Default number of benchmark iterations */ +#define TIMELOOP 2500 /* Minimum timing per iteration */ -#define KB *(1<<10) -#define MB *(1<<20) +#define KB *( 1<<10) +#define MB *( 1<<20) #define GB *(1U<<30) #define MAX_MEM (2 GB - 64 MB) @@ -116,33 +122,32 @@ You can contact the author at : static const char stdinName[] = "-"; -//************************************** -// Display macros -//************************************** +/************************************* +* Display macros +*************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); static unsigned g_displayLevel = 1; -//************************************** -// Unit variables -//************************************** +/************************************* +* Local variables +*************************************/ static int g_nbIterations = NBLOOPS; -static int g_fn_selection = 1; // required within main() & usage() - +static int g_fn_selection = 1; /* required within main() & usage() */ -//********************************************************* -// Benchmark Functions -//********************************************************* +/************************************* +* Benchmark Functions +*************************************/ #if defined(BMK_LEGACY_TIMER) static int BMK_GetMilliStart(void) { - // Based on Legacy ftime() - // Rolls over every ~ 12.1 days (0x100000/24/60/60) - // Use GetMilliSpan to correct for rollover + /* Based on Legacy ftime() + * Rolls over every ~ 12.1 days (0x100000/24/60/60) + * Use GetMilliSpan to correct for rollover */ struct timeb tb; int nCount; ftime( &tb ); @@ -154,8 +159,8 @@ static int BMK_GetMilliStart(void) static int BMK_GetMilliStart(void) { - // Based on newer gettimeofday() - // Use GetMilliSpan to correct for rollover + /* Based on newer gettimeofday() + * Use GetMilliSpan to correct for rollover */ struct timeval tv; int nCount; gettimeofday(&tv, NULL); @@ -174,25 +179,28 @@ static int BMK_GetMilliSpan( int nTimeStart ) } -static size_t BMK_findMaxMem(U64 requestedMem) +static size_t BMK_findMaxMem(U64 requiredMem) { - size_t step = (64 MB); - size_t allocatedMemory; + size_t step = 64 MB; BYTE* testmem=NULL; - requestedMem += 3*step; - requestedMem -= (size_t)requestedMem & (step-1); - if (requestedMem > MAX_MEM) requestedMem = MAX_MEM; - allocatedMemory = (size_t)requestedMem; + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; while (!testmem) { - allocatedMemory -= step; - testmem = (BYTE*) malloc((size_t)allocatedMemory); + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = (BYTE*) malloc ((size_t)requiredMem); } free (testmem); - return (size_t) (allocatedMemory - step); + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; } @@ -206,21 +214,128 @@ static U64 BMK_GetFileSize(char* infilename) struct stat statbuf; r = stat(infilename, &statbuf); #endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ return (U64)statbuf.st_size; } -static int BMK_benchFile(char** fileNamesTable, int nbFiles) +/* Note : buffer is supposed malloc'ed, hence aligned */ +static void BMK_benchMem(const void* buffer, size_t bufferSize) { - int fileIdx=0; - U32 hashResult=0; + static const int nbh_perloop = 100; + + /* XXH32 bench */ + { + int iterationNb; + double fastestH = 100000000.; + U32 hashResult = 0; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + if (g_nbIterations<1) g_nbIterations=1; + for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) + { + int nbHashes = 0; + int milliTime; + + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH32", (int)bufferSize); + + /* Timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)bufferSize, (double)bufferSize / fastestH / 1000., hashResult); + } + + /* Bench XXH32 on Unaligned input */ + { + int iterationNb; + double fastestH = 100000000.; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + for (iterationNb = 1; (iterationNb <= g_nbIterations) && ((bufferSize>1)); iterationNb++) + { + int nbHashes = 0; + int milliTime; + const char* charPtr = (const char*)buffer; + + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "(unaligned)", (int)(bufferSize-1)); + /* timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + } + DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)(bufferSize-1), (double)(bufferSize-1) / fastestH / 1000.); + } + + /* Bench XXH64 */ + { + int iterationNb; + double fastestH = 100000000.; + unsigned long long h64 = 0; + + DISPLAY("\r%79s\r", ""); /* Clean display line */ + for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) + { + int nbHashes = 0; + int milliTime; - U64 totals = 0; - double totalc = 0.; + DISPLAY("%1i-%-14.14s : %10i ->\r", iterationNb, "XXH64", (int)bufferSize); + /* Timing loop */ + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliStart() == milliTime); + milliTime = BMK_GetMilliStart(); + while(BMK_GetMilliSpan(milliTime) < TIMELOOP) + { + int i; + for (i=0; i %7.1f MB/s\r", iterationNb, "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + } + { + DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x", "XXH64", (int)bufferSize, (double)bufferSize / fastestH / 1000.); + DISPLAY("%08X%08X", (U32)(h64 >> 32), (U32)h64); + DISPLAY("\n"); + } + } +} + + +static int BMK_benchFile(char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; - // Loop for each file while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; @@ -256,9 +371,9 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) fclose(inFile); return 12; } - alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries + alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes boundaries */ - // Fill input buffer + /* Fill input buffer */ DISPLAY("\rLoading %s... \n", inFileName); readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); @@ -270,123 +385,39 @@ static int BMK_benchFile(char** fileNamesTable, int nbFiles) return 13; } + /* bench */ + BMK_benchMem(alignedBuffer, benchedSize); - // Bench XXH32 - { - int interationNb; - double fastestC = 100000000.; - - DISPLAY("\r%79s\r", ""); // Clean display line - for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) - { - int nbHashes = 0; - int milliTime; - - DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize); - - // Hash loop - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - hashResult = XXH32(alignedBuffer, benchedSize, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult); - - totals += benchedSize; - totalc += fastestC; - } - - // Bench Unaligned XXH32 - { - int interationNb; - double fastestC = 100000000.; - - DISPLAY("\r%79s\r", ""); // Clean display line - for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++) - { - int nbHashes = 0; - int milliTime; - - DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize); - // Hash loop - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - hashResult = XXH32(alignedBuffer+1, benchedSize-1, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.); - } - - // Bench XXH64 - { - int interationNb; - double fastestC = 100000000.; - unsigned long long h64 = 0; + free(buffer); + } - DISPLAY("\r%79s\r", ""); // Clean display line - for (interationNb = 1; interationNb <= g_nbIterations; interationNb++) - { - int nbHashes = 0; - int milliTime; + return 0; +} - DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize); - // Hash loop - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliStart() == milliTime); - milliTime = BMK_GetMilliStart(); - while(BMK_GetMilliSpan(milliTime) < TIMELOOP) - { - int i; - for (i=0; i<100; i++) - { - h64 = XXH64(alignedBuffer, benchedSize, 0); - nbHashes++; - } - } - milliTime = BMK_GetMilliSpan(milliTime); - if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes; - DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.); - } - DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64)); - totals += benchedSize; - totalc += fastestC; - } +static int BMK_benchInternal(void) +{ + static const size_t benchedSize = 100 KB; + void* buffer; - free(buffer); + buffer = malloc(benchedSize); + if(!buffer) + { + DISPLAY("\nError: not enough memory!\n"); + return 12; } - if (nbFiles > 1) - printf("%-16.16s :%11llu -> %7.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (double)totals/totalc/1000.); + /* bench */ + DISPLAY("\rSample of %u KB... \n", (U32)(benchedSize >> 10)); + BMK_benchMem(buffer, benchedSize); + + free(buffer); return 0; } - static void BMK_checkResult(U32 r1, U32 r2) { static int nbTests = 1; @@ -408,7 +439,7 @@ static void BMK_checkResult64(U64 r1, U64 r2) if (r1!=r2) { DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); - DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2); + DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2); exit(1); } nbTests++; @@ -436,22 +467,22 @@ static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult) } -static void BMK_testSequence(void* sentence, int len, U32 seed, U32 Nresult) +static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult) { U32 Dresult; XXH32_state_t state; - int index; + size_t index; - Dresult = XXH32(sentence, len, seed); + Dresult = XXH32(sequence, len, seed); BMK_checkResult(Dresult, Nresult); XXH32_reset(&state, seed); - XXH32_update(&state, sentence, len); + XXH32_update(&state, sequence, len); Dresult = XXH32_digest(&state); BMK_checkResult(Dresult, Nresult); XXH32_reset(&state, seed); - for (index=0; index>24); - prime *= prime; + sanityBuffer[i] = (BYTE)(byteGen>>24); + byteGen *= byteGen; } BMK_testSequence(NULL, 0, 0, 0x02CC5D05); - BMK_testSequence(NULL, 0, PRIME, 0x36B78AE7); + BMK_testSequence(NULL, 0, prime, 0x36B78AE7); BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5); - BMK_testSequence(sanityBuffer, 1, PRIME, 0xD5845D64); + BMK_testSequence(sanityBuffer, 1, prime, 0xD5845D64); BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4); - BMK_testSequence(sanityBuffer, 14, PRIME, 0x4481951D); + BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D); BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412); - BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2); + BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2); BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL); - BMK_testSequence64(NULL , 0, PRIME, 0xAC75FDA2929B17EFULL); + BMK_testSequence64(NULL , 0, prime, 0xAC75FDA2929B17EFULL); BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL); - BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL); + BMK_testSequence64(sanityBuffer, 1, prime, 0x739840CB819FA723ULL); BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL); - BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL); + BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); - BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL); + BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL); - DISPLAY("\r%79s\r", ""); // Clean display line + DISPLAY("\r%79s\r", ""); /* Clean display line */ DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); } +static void BMK_display_BigEndian(const void* ptr, size_t length) +{ + const BYTE* p = (const BYTE*)ptr; + size_t index = BMK_isLittleEndian() ? length-1 : 0; + int incr = BMK_isLittleEndian() ? -1 : 1; + while (index>32), (U32)(h64), fileName); + BMK_display_BigEndian(&h64, 8); + DISPLAYRESULT(" %s \n", fileName); break; } default: @@ -585,9 +629,9 @@ static int BMK_hash(const char* fileName, U32 hashNb) } -//********************************************************* -// Main -//********************************************************* +/********************************************************* +* Main +*********************************************************/ static int usage(const char* exename) { @@ -615,18 +659,18 @@ static int badusage(const char* exename) int main(int argc, char** argv) { int i, filenamesStart=0; - const char* input_filename = (char*)stdinName; + const char* input_filename = (const char*)stdinName; const char* exename = argv[0]; U32 benchmarkMode = 0; - // xxh32sum default to 32 bits checksum + /* xxh32sum default to 32 bits checksum */ if (strstr(exename, "xxh32sum")!=NULL) g_fn_selection=0; for(i=1; i 1) return badusage(exename); return BMK_hash(input_filename, g_fn_selection);