Skip to content

Commit

Permalink
Merge pull request #136 from Cyan4973/dev
Browse files Browse the repository at this point in the history
xxHash v0.6.5
  • Loading branch information
Cyan4973 authored Apr 18, 2018
2 parents 7b0e264 + 3064d42 commit 7cc9639
Show file tree
Hide file tree
Showing 10 changed files with 1,008 additions and 329 deletions.
2 changes: 0 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,3 @@ before_install:
- sudo apt-get install -qq clang
- sudo apt-get install -qq g++-multilib
- sudo apt-get install -qq gcc-multilib
- sudo apt-get install -qq valgrind

70 changes: 47 additions & 23 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,19 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT))
LIBVER := $(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH)

CFLAGS ?= -O3
# SSE4 detection
HAVE_SSE4 := $(shell $(CC) -dM -E - < /dev/null | grep "SSE4" > /dev/null && echo 1 || echo 0)
ifeq ($(HAVE_SSE4), 1)
NOSSE4 := -mno-sse4
else
NOSSE4 :=
endif

CFLAGS ?= -O2 $(NOSSE4) # disables potential auto-vectorization
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
-Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-Wstrict-prototypes -Wundef
-Wstrict-prototypes -Wundef

FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(MOREFLAGS)
XXHSUM_VERSION=$(LIBVER)
MD2ROFF = ronn
Expand Down Expand Up @@ -67,16 +76,19 @@ LIBXXH = libxxhash.$(SHARED_EXT_VER)


.PHONY: default
default: lib xxhsum
default: lib xxhsum_and_links

.PHONY: all
all: lib xxhsum xxhsum32 xxhsum_inlinedXXH
all: lib xxhsum xxhsum_inlinedXXH

xxhsum32: CFLAGS += -m32
xxhsum xxhsum32: xxhash.c xxhsum.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
ln -sf $@ xxh32sum
ln -sf $@ xxh64sum

.PHONY: xxhsum_and_links
xxhsum_and_links: xxhsum
ln -sf xxhsum xxh32sum
ln -sf xxhsum xxh64sum

xxhsum_inlinedXXH: xxhsum.c
$(CC) $(FLAGS) -DXXH_PRIVATE_API $^ -o $@$(EXT)
Expand All @@ -89,7 +101,10 @@ libxxhash.a: xxhash.o
@echo compiling static library
@$(AR) $(ARFLAGS) $@ $^

$(LIBXXH): LDFLAGS += -shared -fPIC
$(LIBXXH): LDFLAGS += -shared
ifeq (,$(filter Windows%,$(OS)))
$(LIBXXH): LDFLAGS += -fPIC
endif
$(LIBXXH): xxhash.c
@echo compiling dynamic library $(LIBVER)
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
Expand All @@ -104,12 +119,12 @@ lib: libxxhash.a libxxhash

# tests

.PHONY: test
test: xxhsum
.PHONY: check
check: xxhsum
# stdin
./xxhsum < xxhash.c
# multiple files
./xxhsum *
./xxhsum xxhash.* xxhsum.*
# internal bench
./xxhsum -bi1
# file bench
Expand All @@ -119,21 +134,21 @@ test: xxhsum
test-mem: xxhsum
# memory tests
valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -bi1 xxhash.c
valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c
valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c
valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H0 xxhash.c
valgrind --leak-check=yes --error-exitcode=1 ./xxhsum -H1 xxhash.c

.PHONY: test32
test32: clean xxhsum32
@echo ---- test 32-bits ----
@echo ---- test 32-bit ----
./xxhsum32 -bi1 xxhash.c

test-xxhsum-c: xxhsum
# xxhsum to/from pipe
./xxhsum * | ./xxhsum -c -
./xxhsum -H0 * | ./xxhsum -c -
./xxhsum lib* | ./xxhsum -c -
./xxhsum -H0 lib* | ./xxhsum -c -
# xxhsum to/from file, shell redirection
./xxhsum * > .test.xxh64
./xxhsum -H0 * > .test.xxh32
./xxhsum lib* > .test.xxh64
./xxhsum -H0 lib* > .test.xxh32
./xxhsum -c .test.xxh64
./xxhsum -c .test.xxh32
./xxhsum -c < .test.xxh64
Expand All @@ -147,8 +162,6 @@ test-xxhsum-c: xxhsum
# Expects "FAILED open or read"
echo "0000000000000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1

clean-xxhsum-c:
@$(RM) -f .test.xxh32 .test.xxh64

armtest: clean
Expand All @@ -168,9 +181,10 @@ c90test: clean
$(CC) -std=c90 -Werror -pedantic -DXXH_NO_LONG_LONG -c xxhash.c
$(RM) xxhash.o

usan: CC=clang
usan: clean
@echo ---- check undefined behavior - sanitize ----
$(MAKE) clean test CC=clang MOREFLAGS="-g -fsanitize=undefined"
$(MAKE) clean test CC=$(CC) MOREFLAGS="-g -fsanitize=undefined -fno-sanitize-recover=all"

staticAnalyze: clean
@echo ---- static analyzer - scan-build ----
Expand All @@ -193,15 +207,21 @@ clean-man:
preview-man: clean-man man
man ./xxhsum.1

test-all: clean all namespaceTest test test32 test-xxhsum-c clean-xxhsum-c \
armtest clangtest gpptest c90test test-mem usan staticAnalyze
test: all namespaceTest check test-xxhsum-c c90test

test-all: test test32 armtest clangtest gpptest usan listL120 trailingWhitespace staticAnalyze

.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note : $$, for Makefile compatibility)
find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done

.PHONY: trailingWhitespace
trailingWhitespace:
! grep -E "`printf '[ \\t]$$'`" *.1 *.c *.h LICENSE Makefile cmake_unofficial/CMakeLists.txt

.PHONY: clean
clean: clean-xxhsum-c
clean:
@$(RM) -r *.dSYM # Mac OS-X specific
@$(RM) core *.o libxxhash.*
@$(RM) xxhsum$(EXT) xxhsum32$(EXT) xxhsum_inlinedXXH$(EXT) xxh32sum xxh64sum
@echo cleaning completed
Expand All @@ -212,6 +232,10 @@ clean: clean-xxhsum-c
#-----------------------------------------------------------------------------
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))

.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs

DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
Expand Down
35 changes: 21 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Code is highly portable, and hashes are identical on all platforms (little / big
Benchmarks
-------------------------

The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bits box.
The benchmark uses SMHasher speed test, compiled with Visual 2010 on a Windows Seven 32-bit box.
The reference system uses a Core 2 Duo @3GHz


Expand All @@ -40,13 +40,13 @@ It depends on successfully passing SMHasher test set.
Algorithms with a score < 5 are not listed on this table.

A more recent version, XXH64, has been created thanks to [Mathias Westerdahl](https://github.com/JCash),
which offers superior speed and dispersion for 64-bits systems.
Note however that 32-bits applications will still run faster using the 32-bits version.
which offers superior speed and dispersion for 64-bit systems.
Note however that 32-bit applications will still run faster using the 32-bit version.

SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bits.
SMHasher speed test, compiled using GCC 4.8.2, on Linux Mint 64-bit.
The reference system uses a Core i5-3340M @2.7GHz

| Version | Speed on 64-bits | Speed on 32-bits |
| Version | Speed on 64-bit | Speed on 32-bit |
|------------|------------------|------------------|
| XXH64 | 13.8 GB/s | 1.9 GB/s |
| XXH32 | 6.8 GB/s | 6.0 GB/s |
Expand All @@ -66,9 +66,15 @@ The utility `xxhsum` is GPL licensed.
The following macros can be set at compilation time,
they modify xxhash behavior. They are all disabled by default.

- `XXH_INLINE_ALL` : Make all functions `inline`, with bodies directly included within `xxhash.h`.
There is no need for an `xxhash.o` module in this case.
Inlining functions is generally beneficial for speed on small keys.
It's especially effective when key length is a compile time constant,
with observed performance improvement in the +200% range .
See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details.
- `XXH_ACCEPT_NULL_INPUT_POINTER` : if set to `1`, when input is a null-pointer,
xxhash result is the same as a null-length key,
instead of a dereference segfault.
xxhash result is the same as a zero-length key
(instead of a dereference segfault).
- `XXH_FORCE_MEMORY_ACCESS` : default method `0` uses a portable `memcpy()` notation.
Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets.
Method `2` forces unaligned reads, which is not standard compliant, but might sometimes be the only way to extract better performance.
Expand All @@ -77,21 +83,22 @@ they modify xxhash behavior. They are all disabled by default.
Setting it to 0 forces big-endian.
- `XXH_FORCE_NATIVE_FORMAT` : on big-endian systems : use native number representation.
Breaks consistency with little-endian results.
- `XXH_PRIVATE_API` : same impact as `XXH_INLINE_ALL`.
Name underlines that symbols will not be published on library public interface.
- `XXH_NAMESPACE` : prefix all symbols with the value of `XXH_NAMESPACE`.
Useful to evade symbol naming collisions,
in case of multiple inclusions of xxHash library.
Client programs can still use regular function name, symbols are automatically translated through `xxhash.h`.
- `XXH_STATIC_LINKING_ONLY` : gives access to state definition for static allocation.
in case of multiple inclusions of xxHash source code.
Client applications can still use regular function name,
symbols are automatically translated through `xxhash.h`.
- `XXH_STATIC_LINKING_ONLY` : gives access to state declaration for static allocation.
Incompatible with dynamic linking, due to risks of ABI changes.
- `XXH_PRIVATE_API` : Make all functions `static`, directly accessible through `#include xxhash.h`, for inlining.
Do not compile `xxhash.c` as a separate module in this case.
- `XXH_NO_LONG_LONG` : removes support for XXH64,
useful for targets without 64-bits support.
for targets without 64-bit support.


### Example

Calling xxhash 64-bits variant from a C program :
Calling xxhash 64-bit variant from a C program :

```
#include "xxhash.h"
Expand Down
70 changes: 70 additions & 0 deletions appveyor.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
version: 1.0.{build}
environment:
matrix:
- COMPILER: "gcc"
PLATFORM: "mingw64"
- COMPILER: "gcc"
PLATFORM: "mingw32"

install:
- ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
- MKDIR bin
- if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
- if [%COMPILER%]==[gcc] (
SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe
) else (
IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;")
)

build_script:
- if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL%
- ECHO *** &&
ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% &&
ECHO ***
- if [%PLATFORM%]==[clang] (clang -v)
- if [%COMPILER%]==[gcc] (gcc -v)
- if [%COMPILER%]==[gcc] (
echo ----- &&
make -v &&
echo ----- &&
if not [%PLATFORM%]==[clang] (
make -B clean test MOREFLAGS=-Werror
) ELSE (
make -B clean test CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"
)
)
- if [%COMPILER%]==[visual] (
ECHO *** &&
ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /p:EnableWholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
ECHO *** &&
ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
ECHO *** &&
msbuild "visual\VS2010\lz4.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
COPY visual\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe programs\
)

test_script:
- ECHO *** &&
ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% &&
ECHO ***
- if not [%COMPILER%]==[unknown] (
xxhsum -h &&
xxhsum xxhsum.exe &&
xxhsum -bi1 &&
echo ------- xxhsum tested -------
)
Loading

0 comments on commit 7cc9639

Please sign in to comment.