Merge branch 'develop', prepare xz 4.2.0 release

ied206 · Feb 15, 2023 · 363ec35 · 363ec35
2 parents ff8f403 + a48f1e7
commit 363ec35
Show file tree

Hide file tree

Showing 114 changed files with 2,989 additions and 780 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -2,6 +2,9 @@
 # Set default behavior to automatically normalize line endings.
 ###############################################################################
 * text=auto
+# Bash shell script requires LF on POSIX (CRLF is also supported on MSYS2)
+native/**/posix/*.sh text eol=lf
+native/**/windows/*.sh text eol=lf
 
 ###############################################################################
 # Set default behavior for command prompt diff.

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,8 @@
 
 # Joveler Custom
 res/
+native/**/posix/build*/
+native/**/windows/build*/
 
 # User-specific files
 *.suo

diff --git a/Benchmark/BenchProgram.cs b/Benchmark/BenchProgram.cs
@@ -1,16 +1,15 @@
-using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Running;
 using CommandLine;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-// ReSharper disable InconsistentNaming
 
 namespace Benchmark
 {
-    #region Parameter
+    #region Parameters
     [Flags]
     public enum AlgorithmFlags
     {
@@ -26,20 +25,6 @@ public abstract class ParamOptions
     {
         [Option("algo", Default = AlgorithmFlags.All, HelpText = "Choose algorithms to benchmark | zlib,xz,lz4,all")]
         public AlgorithmFlags Algorithms { get; set; }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public T Cast<T>() where T : ParamOptions
-        {
-            T cast = this as T;
-            Debug.Assert(cast != null);
-            return cast;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static T Cast<T>(ParamOptions opts) where T : ParamOptions
-        {
-            return opts.Cast<T>();
-        }
     }
 
     [Verb("all", HelpText = "Benchmark all")]
@@ -51,6 +36,9 @@ public class CompBenchOptions : ParamOptions { }
     [Verb("decomp", HelpText = "Benchmark decompression")]
     public class DecompBenchOptions : ParamOptions { }
 
+    [Verb("xzmulti", HelpText = "Benchmark multithread options (XZ only)")]
+    public class XZMultiOptionBenchOptions : ParamOptions { }
+
     [Verb("hash", HelpText = "Benchmark hash and checksums")]
     public class HashBenchOptions : ParamOptions { }
 
@@ -61,6 +49,11 @@ public class BufferSizeBenchOptions : ParamOptions { }
     #region Program
     public static class Program
     {
+        #region Directories
+        public static string BaseDir => Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", ".."));
+        public static string SampleDir => Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "..", "..", "Samples"));
+        #endregion
+
         #region PrintErrorAndExit
         internal static void PrintErrorAndExit(IEnumerable<Error> errs)
         {
@@ -76,8 +69,6 @@ public static void NativeGlobalInit()
             const string runtimes = "runtimes";
             const string native = "native";
 
-            string baseDir = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", ".."));
-
             string zlibPath = null;
             string xzPath = null;
             string lz4Path = null;
@@ -86,9 +77,9 @@ public static void NativeGlobalInit()
             {
                 string libDir = RuntimeInformation.ProcessArchitecture switch
                 {
-                    Architecture.X86 => Path.Combine(baseDir, runtimes, "win-x86", native),
-                    Architecture.X64 => Path.Combine(baseDir, runtimes, "win-x64", native),
-                    Architecture.Arm64 => Path.Combine(baseDir, runtimes, "win-arm64", native),
+                    Architecture.X86 => Path.Combine(BaseDir, runtimes, "win-x86", native),
+                    Architecture.X64 => Path.Combine(BaseDir, runtimes, "win-x64", native),
+                    Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "win-arm64", native),
                     _ => throw new PlatformNotSupportedException(),
                 };
                 zlibPath = Path.Combine(libDir, "zlibwapi.dll");
@@ -100,9 +91,9 @@ public static void NativeGlobalInit()
             {
                 string libDir = RuntimeInformation.ProcessArchitecture switch
                 {
-                    Architecture.X64 => Path.Combine(baseDir, runtimes, "linux-x64", native),
-                    Architecture.Arm => Path.Combine(baseDir, runtimes, "linux-arm", native),
-                    Architecture.Arm64 => Path.Combine(baseDir, runtimes, "linux-arm64", native),
+                    Architecture.X64 => Path.Combine(BaseDir, runtimes, "linux-x64", native),
+                    Architecture.Arm => Path.Combine(BaseDir, runtimes, "linux-arm", native),
+                    Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "linux-arm64", native),
                     _ => throw new PlatformNotSupportedException(),
                 };
                 zlibPath = Path.Combine(libDir, "libz.so");
@@ -114,8 +105,8 @@ public static void NativeGlobalInit()
             {
                 string libDir = RuntimeInformation.ProcessArchitecture switch
                 {
-                    Architecture.X64 => Path.Combine(baseDir, runtimes, "osx-x64", native),
-                    Architecture.Arm64 => throw new PlatformNotSupportedException("TODO"),
+                    Architecture.X64 => Path.Combine(BaseDir, runtimes, "osx-x64", native),
+                    Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "osx-arm64", native),
                     _ => throw new PlatformNotSupportedException(),
                 };
                 zlibPath = Path.Combine(libDir, "libz.dylib");
@@ -157,34 +148,47 @@ public static void Main(string[] args)
             });
 
             argParser.ParseArguments<AllBenchOptions,
-                CompBenchOptions, DecompBenchOptions, HashBenchOptions, BufferSizeBenchOptions>(args)
+                CompBenchOptions, DecompBenchOptions, XZMultiOptionBenchOptions, HashBenchOptions, BufferSizeBenchOptions>(args)
                 .WithParsed<AllBenchOptions>(x => Opts = x)
                 .WithParsed<CompBenchOptions>(x => Opts = x)
                 .WithParsed<DecompBenchOptions>(x => Opts = x)
+                .WithParsed<XZMultiOptionBenchOptions>(x => Opts = x)
                 .WithParsed<HashBenchOptions>(x => Opts = x)
                 .WithParsed<BufferSizeBenchOptions>(x => Opts = x)
                 .WithNotParsed(PrintErrorAndExit);
             Debug.Assert(Opts != null, $"{nameof(Opts)} != null");
 
+            // InvertedTomato.Crc is the slowest, and ships unoptimized binaries.
+            // Disable for awhile to avoid BenchmarkDotNet's unoptimized run error.
+#if INVERTEDTOMATO_CRC_EANBLE
+            ManualConfig config = DefaultConfig.Instance.WithOptions(ConfigOptions.DisableOptimizationsValidator);
+#else
+            ManualConfig config = DefaultConfig.Instance.WithOptions(ConfigOptions.Default);
+#endif
+
             switch (Opts)
             {
                 case AllBenchOptions _:
-                    BenchmarkRunner.Run<CompBench>();
-                    BenchmarkRunner.Run<DecompBench>();
-                    BenchmarkRunner.Run<HashBench>();
-                    BenchmarkRunner.Run<BufferSizeBench>();
+                    BenchmarkRunner.Run<CompBench>(config);
+                    BenchmarkRunner.Run<DecompBench>(config);
+                    BenchmarkRunner.Run<XZMultiOptionBench>(config);
+                    BenchmarkRunner.Run<HashBench>(config);
+                    BenchmarkRunner.Run<BufferSizeBench>(config);
                     break;
                 case CompBenchOptions _:
-                    BenchmarkRunner.Run<CompBench>();
+                    BenchmarkRunner.Run<CompBench>(config);
                     break;
                 case DecompBenchOptions _:
-                    BenchmarkRunner.Run<DecompBench>();
+                    BenchmarkRunner.Run<DecompBench>(config);
+                    break;
+                case XZMultiOptionBenchOptions _:
+                    BenchmarkRunner.Run<XZMultiOptionBench>(config);
                     break;
                 case HashBenchOptions _:
-                    BenchmarkRunner.Run<HashBench>();
+                    BenchmarkRunner.Run<HashBench>(config);
                     break;
                 case BufferSizeBenchOptions _:
-                    BenchmarkRunner.Run<BufferSizeBench>();
+                    BenchmarkRunner.Run<BufferSizeBench>(config);
                     break;
             }
         }

diff --git a/Benchmark/Benchmark.csproj b/Benchmark/Benchmark.csproj
@@ -7,14 +7,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.3" />
     <PackageReference Include="Crc32.NET" Version="1.2.0" />
-    <PackageReference Include="InvertedTomato.Crc" Version="1.2.0" />
-    <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.2.16" />
+    <PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.3.5" />
     <PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
     <PackageReference Include="K4os.Hash.xxHash" Version="1.0.8" />
     <PackageReference Include="SharpCompress" Version="0.32.2" />
-    <PackageReference Include="ZstdSharp.Port" Version="0.6.6" />
+    <PackageReference Include="ZstdSharp.Port" Version="0.6.7" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/Benchmark/CompBench.cs b/Benchmark/CompBench.cs
@@ -218,7 +218,7 @@ public double ZLib_Managed()
 
         [Benchmark]
         [BenchmarkCategory(BenchConfig.XZ)]
-        public double XZ_Native()
+        public double XZ_Native_Single()
         {
             long compLen;
             byte[] rawData = SrcFiles[SrcFileName];
@@ -244,6 +244,44 @@ public double XZ_Native()
             return CompRatio;
         }
 
+        [Benchmark]
+        [BenchmarkCategory(BenchConfig.XZ)]
+        public double XZ_Native_Multi()
+        {
+            // LZMA2 threaded compression with -9 option takes a lot of memory.
+            // To prevent memory starvation, skip threaded -9 compression.
+            if (Level.Equals("Best", StringComparison.OrdinalIgnoreCase))
+                return 0;
+
+            long compLen;
+            byte[] rawData = SrcFiles[SrcFileName];
+            using (MemoryStream ms = new MemoryStream())
+            {
+                Joveler.Compression.XZ.XZCompressOptions compOpts = new Joveler.Compression.XZ.XZCompressOptions
+                {
+                    Level = XZLevelDict[Level],
+                    LeaveOpen = true,
+                };    
+
+                Joveler.Compression.XZ.XZThreadedCompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedCompressOptions
+                {
+                    Threads = Environment.ProcessorCount,
+                };
+
+                using (MemoryStream rms = new MemoryStream(rawData))
+                using (Joveler.Compression.XZ.XZStream xzs = new Joveler.Compression.XZ.XZStream(ms, compOpts, threadOpts))
+                {
+                    rms.CopyTo(xzs);
+                }
+
+                ms.Flush();
+                compLen = ms.Position;
+            }
+
+            CompRatio = (double)compLen / rawData.Length;
+            return CompRatio;
+        }
+
         [Benchmark]
         [BenchmarkCategory(BenchConfig.ZSTD)]
         public double ZSTD_Native()

diff --git a/Benchmark/DecompBench.cs b/Benchmark/DecompBench.cs
@@ -141,7 +141,7 @@ public long ZLib_Managed()
 
         [Benchmark]
         [BenchmarkCategory(BenchConfig.XZ)]
-        public long XZ_Native()
+        public long XZ_Native_Single()
         {
             byte[] compData = SrcFiles[$"{Level}_{SrcFileName}.xz"];
             using MemoryStream ms = new MemoryStream();
@@ -156,6 +156,27 @@ public long XZ_Native()
             return ms.Length;
         }
 
+        [Benchmark]
+        [BenchmarkCategory(BenchConfig.XZ)]
+        public long XZ_Native_Multi()
+        {
+            byte[] compData = SrcFiles[$"{Level}_{SrcFileName}.xz"];
+            using MemoryStream ms = new MemoryStream();
+            Joveler.Compression.XZ.XZDecompressOptions decompOpts = new Joveler.Compression.XZ.XZDecompressOptions();
+            Joveler.Compression.XZ.XZThreadedDecompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedDecompressOptions()
+            {
+                Threads = Environment.ProcessorCount,
+            };
+            using (MemoryStream rms = new MemoryStream(compData))
+            using (Joveler.Compression.XZ.XZStream zs = new Joveler.Compression.XZ.XZStream(rms, decompOpts, threadOpts))
+            {
+                zs.CopyTo(ms);
+            }
+
+            ms.Flush();
+            return ms.Length;
+        }
+
         [Benchmark]
         [BenchmarkCategory(BenchConfig.XZ)]
         public long XZ_Managed()

diff --git a/Benchmark/HashBench.cs b/Benchmark/HashBench.cs
@@ -117,7 +117,7 @@ public uint CRC32_XZNative()
         }
 
         [Benchmark]
-        [BenchmarkCategory("ZLib", "XZ")]
+        [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)]
         public byte[] CRC32_ForceManaged()
         {
             byte[] compData = SrcFiles[SrcFileName];
@@ -126,24 +126,28 @@ public byte[] CRC32_ForceManaged()
         }
 
         [Benchmark]
-        [BenchmarkCategory("ZLib", "XZ")]
+        [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)]
         public uint CRC32_K4osManaged()
         {
             byte[] compData = SrcFiles[SrcFileName];
             K4os.Hash.Crc.Crc32 crc32 = new K4os.Hash.Crc.Crc32();
             crc32.Update(compData);
             return crc32.Digest();
         }
-
+
+        // InvertedTomato.Crc is the slowest, and ships unoptimized binaries.
+        // Disable for awhile to avoid BenchmarkDotNet's unoptimized run error.
+#if INVERTEDTOMATO_CRC_EANBLE
         [Benchmark]
-        [BenchmarkCategory("ZLib", "XZ")]
+        [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)]
         public ulong CRC32_TomatoManaged()
         {
             byte[] compData = SrcFiles[SrcFileName];
             InvertedTomato.IO.Crc crc32 = InvertedTomato.IO.CrcAlgorithm.CreateCrc32();
             crc32.Append(compData);
             return crc32.Check;
         }
+#endif
         #endregion
 
         #region CRC64

diff --git a/Benchmark/Samples/Best/Banner.bmp.xz b/Benchmark/Samples/Best/Banner.bmp.xz
diff --git a/Benchmark/Samples/Best/Banner.svg.xz b/Benchmark/Samples/Best/Banner.svg.xz
diff --git a/Benchmark/Samples/Best/Type4.txt.xz b/Benchmark/Samples/Best/Type4.txt.xz
diff --git a/Benchmark/Samples/Default/Banner.bmp.xz b/Benchmark/Samples/Default/Banner.bmp.xz
diff --git a/Benchmark/Samples/Default/Banner.svg.xz b/Benchmark/Samples/Default/Banner.svg.xz
diff --git a/Benchmark/Samples/Default/Type4.txt.xz b/Benchmark/Samples/Default/Type4.txt.xz
diff --git a/Benchmark/Samples/Fastest/Banner.bmp.xz b/Benchmark/Samples/Fastest/Banner.bmp.xz
diff --git a/Benchmark/Samples/Fastest/Banner.svg.xz b/Benchmark/Samples/Fastest/Banner.svg.xz
diff --git a/Benchmark/Samples/Fastest/Type4.txt.xz b/Benchmark/Samples/Fastest/Type4.txt.xz