Merge pull request #186 from Cyan4973/dev

v0.6.1

.gitattributes

@@ -8,8 +8,9 @@
 # Denote files that should not be modified.
 *.odt binary
 *.png binary
+
 # Visual Studio
-*.sln binary
+*.sln text eol=crlf
+*.vcxproj* text eol=crlf
 *.suo binary
-*.vcxproj* binary
-
+*.rc binary

.gitignore

@@ -17,33 +17,17 @@
 *.out
 *.app
 
-# Visual solution files
-*.suo
-*.user
-*.VC.db
-
-# Build results
-[Dd]ebug/
-[Rr]elease/
-[Rr]eleases/
-x64/
-x86/
-[Bb]in/
-[Oo]bj/
-
-# Visual C++ cache files
-ipch/
-*.aps
-*.ncb
-*.opendb
-*.opensdf
-*.sdf
-*.cachefile
+# Visual C++
+visual/
 
 # IDEA solution files
 *.idea
 
 # Other files
 .directory
-_codelite
-_zstdbench
+_codelite/
+_zstdbench/
+zlib_wrapper/
+
+# CMake
+contrib/cmake/

.travis.yml

@@ -12,6 +12,7 @@ env:
   - ZSTD_TRAVIS_CI_ENV=cmaketest
   - ZSTD_TRAVIS_CI_ENV=clangtest  
   - ZSTD_TRAVIS_CI_ENV=gpptest  
+  - ZSTD_TRAVIS_CI_ENV=gnu90test
   - ZSTD_TRAVIS_CI_ENV=armtest-w-install  
   - ZSTD_TRAVIS_CI_ENV=test  
   - ZSTD_TRAVIS_CI_ENV="-C programs test32"  

Makefile

@@ -31,7 +31,7 @@
 # ################################################################
 
 # force a version number : uncomment below export (otherwise, default to the one declared into zstd.h)
-#export VERSION := 0.5.1
+#export VERSION := 0.6.1
 
 PRGDIR  = programs
 ZSTDDIR = lib
@@ -54,6 +54,9 @@ all:
 zstdprogram:
 	$(MAKE) -C $(PRGDIR)
 
+test:
+	$(MAKE) -C $(PRGDIR) $@
+
 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@@ -75,9 +78,6 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
-test:
-	$(MAKE) -C $(PRGDIR) $@
-
 cmaketest:
 	cd contrib/cmake ; cmake . ; $(MAKE)
 
@@ -88,6 +88,21 @@ clangtest: clean
 gpptest: clean
 	$(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+gnu90test: clean
+	$(MAKE) all CFLAGS="-std=gnu90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Wdeclaration-after-statement -Werror"
+
+c90test: clean
+	$(MAKE) all CFLAGS="-std=c90 -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef -Werror"   # will fail, due to // and long long
+
+bmix64test: clean
+	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test
+
+bmix32test: clean
+	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test
+
+bmi32test: clean
+	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
+
 armtest: clean
 	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
 	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"

NEWS

@@ -1,3 +1,13 @@
+v0.6.1
+New : zlib wrapper API, thanks to Przemyslaw Skibinski
+New : Ability to compile compressor / decompressor separately
+Changed : new lib directory structure
+Fixed : Legacy codec v0.5 compatible with dictionary decompression
+Fixed : Decoder corruption error (#173)
+Fixed : null-string roundtrip (#176)
+New : benchmark mode can select directory as input
+Experimental : midipix support, VMS support
+
 v0.6.0
 Stronger high compression modes, thanks to Przemyslaw Skibinski
 API : ZSTD_getFrameParams() provides size of decompressed content
@@ -95,7 +105,7 @@ frame concatenation support
 v0.1.1
 fix compression bug
 detects write-flush errors
-
+git@github.com:Cyan4973/zstd.git
 v0.1.0
 first release
 

appveyor.yml

@@ -0,0 +1,25 @@
+version: 1.0.{build}
+configuration:
+- Release
+- Debug
+platform:
+- Win32
+- x64
+environment:
+  matrix:
+# - PlatformToolset: v100
+  - PlatformToolset: v110
+  - PlatformToolset: v120
+  - PlatformToolset: v140
+build_script:
+- cmd: >-
+    ECHO PlatformToolset=%PlatformToolset%
+
+    msbuild "visual\2013\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=%PlatformToolset% /t:Clean,Build /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll"
+test_script:
+- cmd: >-
+    ECHO %APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%
+
+    SET FUZZERTEST=-T2mn
+
+    IF %CONFIGURATION%==Release (%APPVEYOR_BUILD_FOLDER%\visual\2013\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe %FUZZERTEST%)

contrib/cmake/lib/CMakeLists.txt

@@ -47,40 +47,45 @@ SET(ROOT_DIR ../../..)
 
 # Define library directory, where sources and header files are located
 SET(LIBRARY_DIR ${ROOT_DIR}/lib)
-INCLUDE_DIRECTORIES(${LIBRARY_DIR})
+INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common)
 
 # Read file content
-FILE(READ ${LIBRARY_DIR}/zstd.h HEADER_CONTENT)
+FILE(READ ${LIBRARY_DIR}/common/zstd.h HEADER_CONTENT)
 
 # Parse version
 GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
 MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
 
 SET(Sources
-        ${LIBRARY_DIR}/divsufsort.c
-        ${LIBRARY_DIR}/fse.c
-        ${LIBRARY_DIR}/huff0.c
-        ${LIBRARY_DIR}/zbuff.c
-        ${LIBRARY_DIR}/zdict.c
-        ${LIBRARY_DIR}/zstd_compress.c
-        ${LIBRARY_DIR}/zstd_decompress.c)
+        ${LIBRARY_DIR}/common/entropy_common.c
+        ${LIBRARY_DIR}/common/zstd_common.c
+        ${LIBRARY_DIR}/common/fse_decompress.c
+        ${LIBRARY_DIR}/compress/fse_compress.c
+        ${LIBRARY_DIR}/compress/huf_compress.c
+        ${LIBRARY_DIR}/compress/zbuff_compress.c
+        ${LIBRARY_DIR}/compress/zstd_compress.c
+        ${LIBRARY_DIR}/decompress/huf_decompress.c
+        ${LIBRARY_DIR}/decompress/zbuff_decompress.c
+        ${LIBRARY_DIR}/decompress/zstd_decompress.c
+        ${LIBRARY_DIR}/dictBuilder/divsufsort.c
+        ${LIBRARY_DIR}/dictBuilder/zdict.c)
 
 SET(Headers
-        ${LIBRARY_DIR}/bitstream.h
-        ${LIBRARY_DIR}/error_private.h
-        ${LIBRARY_DIR}/error_public.h
-        ${LIBRARY_DIR}/fse.h
-        ${LIBRARY_DIR}/fse_static.h
-        ${LIBRARY_DIR}/huff0.h
-        ${LIBRARY_DIR}/huff0_static.h
-        ${LIBRARY_DIR}/mem.h
-        ${LIBRARY_DIR}/zbuff.h
-        ${LIBRARY_DIR}/zbuff_static.h
-        ${LIBRARY_DIR}/zdict.h
-        ${LIBRARY_DIR}/zdict_static.h
-        ${LIBRARY_DIR}/zstd_internal.h
-        ${LIBRARY_DIR}/zstd_static.h
-        ${LIBRARY_DIR}/zstd.h)
+        ${LIBRARY_DIR}/common/bitstream.h
+        ${LIBRARY_DIR}/common/error_private.h
+        ${LIBRARY_DIR}/common/error_public.h
+        ${LIBRARY_DIR}/common/fse.h
+        ${LIBRARY_DIR}/common/fse_static.h
+        ${LIBRARY_DIR}/common/huf.h
+        ${LIBRARY_DIR}/common/huf_static.h
+        ${LIBRARY_DIR}/common/mem.h
+        ${LIBRARY_DIR}/common/zbuff.h
+        ${LIBRARY_DIR}/common/zbuff_static.h
+        ${LIBRARY_DIR}/common/zstd_internal.h
+        ${LIBRARY_DIR}/common/zstd_static.h
+        ${LIBRARY_DIR}/common/zstd.h
+        ${LIBRARY_DIR}/dictBuilder/zdict.h
+        ${LIBRARY_DIR}/dictBuilder/zdict_static.h)
 
 IF (ZSTD_LEGACY_SUPPORT)
     SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@@ -117,8 +122,8 @@ ENDIF (MSVC)
 
 # Define include directories
 IF (NOT WORKAROUND_OUTDATED_CODE_STYLE)
-    TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR})
-    TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR})
+    TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_DIR}/common)
+    TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_DIR}/common)
     IF (ZSTD_LEGACY_SUPPORT)
         TARGET_INCLUDE_DIRECTORIES(libzstd_static PUBLIC ${LIBRARY_LEGACY_DIR})
         TARGET_INCLUDE_DIRECTORIES(libzstd_shared PUBLIC ${LIBRARY_LEGACY_DIR})

contrib/cmake/programs/CMakeLists.txt

@@ -44,7 +44,7 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR})
 IF (WORKAROUND_OUTDATED_CODE_STYLE)
     # Define library directory, where sources and header files are located
     SET(LIBRARY_DIR ${ROOT_DIR}/lib)
-    INCLUDE_DIRECTORIES(${LIBRARY_DIR})
+    INCLUDE_DIRECTORIES(${LIBRARY_DIR}/common)
 ENDIF (WORKAROUND_OUTDATED_CODE_STYLE)
 
 IF (ZSTD_LEGACY_SUPPORT)

lib/Makefile

@@ -31,9 +31,9 @@
 # ################################################################
 
 # Version numbers
-LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
-LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
-LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < zstd.h`
+LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h`
+LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h`
+LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./common/zstd.h`
 LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT)
 LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT))
 LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT))
@@ -43,15 +43,18 @@ VERSION?= $(LIBVER)
 
 DESTDIR?=
 PREFIX ?= /usr/local
-CPPFLAGS= -I.
+CPPFLAGS= -I./common
 CFLAGS ?= -O3
 CFLAGS += -std=c99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -Wstrict-aliasing=1
 FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
 
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR=$(PREFIX)/include
 
-ZSTD_FILES := zstd_compress.c zstd_decompress.c fse.c huff0.c zbuff.c zdict.c divsufsort.c
+ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c 
+ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c
+ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c
+ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES)
 ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
 
 ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
@@ -117,9 +120,9 @@ install: libzstd libzstd.pc
 	@cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
 	@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
 	@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
-	@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
-	@install -m 644 zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
-	@install -m 644 zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
+	@install -m 644 common/zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
+	@install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+	@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
 	@echo zstd static and shared library installed
 
 uninstall:

lib/README.md

@@ -5,53 +5,61 @@ The __lib__ directory contains several files, but depending on target use case,
 
 #### Minimal library files
 
-##### Shared ressources
+To build the zstd library the following files are required:
+
+- [common/bitstream.h](common/bitstream.h)
+- [common/error_private.h](common/error_private.h)
+- [common/error_public.h](common/error_public.h)
+- common/fse.h
+- common/fse_decompress.c
+- common/fse_static.h
+- common/huf.h
+- common/huf_static.h
+- [common/mem.h](common/mem.h)
+- [common/zstd.h]
+- common/zstd_internal.h
+- common/zstd_static.h
+- compress/fse_compress.c
+- compress/huf_compress.c
+- compress/zstd_compress.c
+- compress/zstd_opt.h 
+- decompress/huf_decompress.c
+- decompress/zstd_decompress.c
+
+Stable API is exposed in [common/zstd.h].
+Advanced and experimental API is exposed in [common/zstd_static.h].
+API elements of [common/zstd_static.h] should be used with static linking only,
+as their definition may change in future version of the library.
 
-- [mem.h](mem.h)
-- [error_private.h](error_private.h)
-- [error_public.h](error_public.h)
+[common/zstd.h]: common/zstd.h
+[common/zstd_static.h]: common/zstd_static.h
 
-##### zstd core compression
 
-Stable API is exposed in [zstd.h].
-Advanced and experimental API is exposed in `zstd_static.h`.
-`zstd_static.h` API elements should be used with static linking only,
-as their definition may change in future version of the library.
+#### Separate compressor and decompressor
+
+To build a separate zstd compressor all files from `common/` and `compressor/` directories are required.
+In a similar way to build a separate zstd decompressor all files from `common/` and `decompressor/` directories are needed.
 
-- [bitstream.h](bitstream.h)
-- fse.c
-- fse.h
-- fse_static.h
-- huff0.c
-- huff0.h
-- huff0_static.h
-- zstd_compress.c
-- zstd_decompress.c
-- zstd_internal.h
-- zstd_opt.h
-- [zstd.h]
-- zstd_static.h
-
-[zstd.h]: zstd.h
 
 #### Buffered streaming
 
 This complementary API makes streaming integration easier.
 It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/projects/7-Zip-ZStd) :
 
-- zbuff.c
-- zbuff.h
-- zbuff_static.h
+- common/zbuff.h
+- common/zbuff_static.h
+- compress/zbuff_compress.c
+- decompress/zbuff_decompress.c
 
 #### Dictionary builder
 
 To create dictionaries from training sets :
 
-- divsufsort.c
-- divsufsort.h
-- zdict.c
-- zdict.h
-- zdict_static.h
+- dictBuilder/divsufsort.c
+- dictBuilder/divsufsort.h
+- dictBuilder/zdict.c
+- dictBuilder/zdict.h
+- dictBuilder/zdict_static.h
 
 #### Miscellaneous