Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
ied206 committed Aug 6, 2023
2 parents 308da80 + 534122d commit 0b0b9c0
Show file tree
Hide file tree
Showing 32 changed files with 93 additions and 199 deletions.
6 changes: 3 additions & 3 deletions Benchmark/Benchmark.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.13.3" />
<PackageReference Include="BenchmarkDotNet" Version="0.13.6" />
<PackageReference Include="Crc32.NET" Version="1.2.0" />
<PackageReference Include="K4os.Compression.LZ4.Streams" Version="1.3.5" />
<PackageReference Include="K4os.Hash.Crc" Version="1.1.4" />
<PackageReference Include="K4os.Hash.xxHash" Version="1.0.8" />
<PackageReference Include="SharpCompress" Version="0.32.2" />
<PackageReference Include="ZstdSharp.Port" Version="0.6.7" />
<PackageReference Include="SharpCompress" Version="0.33.0" />
<PackageReference Include="ZstdSharp.Port" Version="0.7.2" />
</ItemGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
</ItemGroup>

<ItemGroup>
Expand Down
36 changes: 8 additions & 28 deletions Joveler.Compression.LZ4.Tests/TestSetup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
using System.IO;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
// ReSharper disable InconsistentNaming

namespace Joveler.Compression.LZ4.Tests
Expand Down Expand Up @@ -112,37 +113,16 @@ private static string GetNativeLibPath()
return libPath;
}

#region LogEnvironment
[TestMethod]
public void LogRuntimeInfo()
public void LogEnvironment()
{
string platform = "unknown";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
platform = "win";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
platform = "linux";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
platform = "osx";

string arch = "unknown";
switch (RuntimeInformation.ProcessArchitecture)
{
case Architecture.X86:
arch = "x86";
break;
case Architecture.X64:
arch = "x64";
break;
case Architecture.Arm:
arch = "arm";
break;
case Architecture.Arm64:
arch = "arm64";
break;
}

Console.WriteLine($"Platform = {platform}");
Console.WriteLine($"Arch = {arch}");
StringBuilder b = new StringBuilder();
b.AppendLine($"OS = {RuntimeInformation.OSDescription} {RuntimeInformation.OSArchitecture}");
b.AppendLine($"Dotnet Runtime = {RuntimeInformation.FrameworkDescription} {RuntimeInformation.ProcessArchitecture}");
Console.WriteLine(b.ToString());
}
#endregion
}

public static class TestHelper
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
<PackageReference Include="coverlet.collector" Version="3.2.0">
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
<PackageReference Include="coverlet.collector" Version="6.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
Expand Down
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.elf
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.exe
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.mach
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.armhf.elf
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.elf
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.exe
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.mach
Binary file not shown.
Binary file modified Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x86.exe
Binary file not shown.
36 changes: 8 additions & 28 deletions Joveler.Compression.XZ.Tests/TestSetup.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;

namespace Joveler.Compression.XZ.Tests
{
Expand Down Expand Up @@ -106,37 +107,16 @@ private static string GetNativeLibPath()
return libPath;
}

#region LogEnvironment
[TestMethod]
public void LogRuntimeInfo()
public void LogEnvironment()
{
string platform = "unknown";
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
platform = "win";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
platform = "linux";
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
platform = "osx";

string arch = "unknown";
switch (RuntimeInformation.ProcessArchitecture)
{
case Architecture.X86:
arch = "x86";
break;
case Architecture.X64:
arch = "x64";
break;
case Architecture.Arm:
arch = "arm";
break;
case Architecture.Arm64:
arch = "arm64";
break;
}

Console.WriteLine($"Platform = {platform}");
Console.WriteLine($"Arch = {arch}");
StringBuilder b = new StringBuilder();
b.AppendLine($"OS = {RuntimeInformation.OSDescription} {RuntimeInformation.OSArchitecture}");
b.AppendLine($"Dotnet Runtime = {RuntimeInformation.FrameworkDescription} {RuntimeInformation.ProcessArchitecture}");
Console.WriteLine(b.ToString());
}
#endregion
}

public static class TestHelper
Expand Down
6 changes: 6 additions & 0 deletions Joveler.Compression.XZ/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

## v4.x

### v4.2.2

Released in 2023-08-06

- Update xz-utils to 5.4.3.

### v4.2.1

Released in 2023-02-16
Expand Down
6 changes: 2 additions & 4 deletions Joveler.Compression.XZ/Joveler.Compression.XZ.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

<PackageId>Joveler.Compression.XZ</PackageId>
<Title>Joveler.Compression.XZ</Title>
<Version>4.2.1</Version>
<Version>4.2.2</Version>
<Authors>Hajin Jang</Authors>
<Company>Joveler</Company>
<Description>XZ Utils (liblzma) pinvoke library for .NET.
Expand All @@ -21,9 +21,7 @@ Supports Windows, Linux and macOS.</Description>
<PackageProjectUrl>https://github.com/ied206/Joveler.Compression</PackageProjectUrl>
<PackageIcon>images\Logo.png</PackageIcon>
<RepositoryUrl>https://github.com/ied206/Joveler.Compression</RepositoryUrl>
<PackageReleaseNotes>- Support xz-utils 5.4.1.
- Target .NET Framework 4.6 instead of deprecated 4.5.1.
- Fix .NET Framework build script path issue.</PackageReleaseNotes>
<PackageReleaseNotes>- Update xz-utils to 5.4.3.</PackageReleaseNotes>
<PackageTags>xz xz-utils liblzma lzma lzma2 compress compression decompress decompression archive crc32 crc64 crc checksum native pinvoke interop</PackageTags>
<PackageReadmeFile>README.md</PackageReadmeFile>
</PropertyGroup>
Expand Down
11 changes: 2 additions & 9 deletions Joveler.Compression.XZ/NUGET_README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
## Features

- XZStream, the stream for [.xz file format](https://tukaani.org/xz/xz-file-format.txt).
- Fast native implementation of CRC32 and CRC64 checksums.

## Usage

Expand Down Expand Up @@ -38,15 +39,7 @@ Please refer the [project homepage](https://github.com/ied206/Joveler.Compressio
| macOS | x64 | Yes |
| | arm64 | Yes |

#### Tested linux distributions

| Architecture | Distribution | Note |
|---------------|--------------|------|
| x64 | Ubuntu 20.04 | Tested on AppVeyor CI |
| armhf | Debian 11 | Emulated on QEMU |
| arm64 | Debian 11 | Emulated on QEMU |

### Supported XZ Utils version

- 5.4.1 (Included)
- 5.4.3 (Included)

11 changes: 2 additions & 9 deletions Joveler.Compression.XZ/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Joveler.Compression.XZ can be installed via [nuget](https://www.nuget.org/packag
## Features

- XZStream, the stream for [.xz file format](https://tukaani.org/xz/xz-file-format.txt).
- Fast native implementation of CRC32 and CRC64 checksums.

## Support

Expand Down Expand Up @@ -40,17 +41,9 @@ Joveler.Compression.XZ can be installed via [nuget](https://www.nuget.org/packag
| macOS | x64 | Yes |
| | arm64 | Yes |

#### Tested linux distributions

| Architecture | Distribution | Note |
|---------------|--------------|------|
| x64 | Ubuntu 20.04 | Tested on AppVeyor CI |
| armhf | Debian 11 | Emulated on QEMU |
| arm64 | Debian 11 | Emulated on QEMU |

### Supported XZ Utils version

- 5.4.1 (Included)
- 5.4.3 (Included)

## Usage

Expand Down
64 changes: 24 additions & 40 deletions Joveler.Compression.XZ/USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ You must call `XZInit.GlobalInit()` before using Joveler.Compression.XZ. Please

**WARNING**: The caller process and callee library must have the same architecture!

#### On .NET Core
#### On .NET/.NET Core

```cs
public static void InitNativeLibrary()
Expand Down Expand Up @@ -89,39 +89,25 @@ public static void InitNativeLibrary()

### Embedded binary

Joveler.Compression.XZ comes with sets of static binaries of `liblzma 5.4.1`. They will be copied into the build directory at build time.
Joveler.Compression.XZ comes with sets of static binaries of `liblzma 5.4.3`. They will be copied into the build directory at build time.

#### On .NET Standard & .NET Core
#### On .NET/.NET Core & .NET Standard

| Platform | Binary | License | Note |
|----------------------|----------------------------------------------|---------------|---------------|
| Windows x86 | `$(OutDir)\runtimes\win-x86\liblzma.dll` | Public Domain | Universal CRT |
| Windows x64 | `$(OutDir)\runtimes\win-x64\liblzma.dll` | Public Domain | Universal CRT |
| Windows arm64 | `$(OutDir)\runtimes\win-arm64\liblzma.dll` | Public Domain | Universal CRT |
| Ubuntu 20.04 x64 | `$(OutDir)\runtimes\linux-x64\liblzma.so` | Public Domain | glibc |
| Debian 11 armhf | `$(OutDir)\runtimes\linux-arm\liblzma.so` | Public Domain | glibc |
| Debian 11 arm64 | `$(OutDir)\runtimes\linux-arm64\liblzma.so` | Public Domain | glibc |
| macOS Big Sur x64 | `$(OutDir)\runtimes\osx-x64\liblzma.dylib` | Public Domain | libSystem |
| macOS Ventura arm64 | `$(OutDir)\runtimes\osx-arm64\liblzma.dylib` | Public Domain | libSystem |

- Create an empty file named `Joveler.Compression.XZ.Precompiled.Exclude` in the project directory to prevent copy of the package-embedded binary.

#### For .NET Standard 2.0+

| Platform | Binary | Note |
|---------------|---------------------------------------------|--------------------------|
| Windows x86 | `$(OutDir)\runtimes\win-x86\liblzma.dll` | Official binary |
| Windows x64 | `$(OutDir)\runtimes\win-x64\liblzma.dll` | Official binary |
| Windows arm64 | `$(OutDir)\runtimes\win-arm64\liblzma.dll` | Compiled with MSVC 2019 |
| Linux x64 | `$(OutDir)\runtimes\linux-x64\liblzma.so` | Compiled in Ubuntu 18.04 |
| Linux armhf | `$(OutDir)\runtimes\linux-arm\liblzma.so` | Compiled in Debian 10 |
| Linux arm64 | `$(OutDir)\runtimes\linux-arm64\liblzma.so` | Compiled in Debian 10 |
| macOS x64 | `$(OutDir)\runtimes\osx-x64\liblzma.dylib` | Compiled in Catalina |
| Platform | Binary | License | C Runtime |
|-----------------------|----------------------------------------------|---------------|---------------|
| Windows x86 | `$(OutDir)\runtimes\win-x86\liblzma.dll` | Public Domain | Universal CRT |
| Windows x64 | `$(OutDir)\runtimes\win-x64\liblzma.dll` | Public Domain | Universal CRT |
| Windows arm64 | `$(OutDir)\runtimes\win-arm64\liblzma.dll` | Public Domain | Universal CRT |
| Ubuntu 20.04 x64 | `$(OutDir)\runtimes\linux-x64\liblzma.so` | Public Domain | glibc |
| Debian 12 armhf | `$(OutDir)\runtimes\linux-arm\liblzma.so` | Public Domain | glibc |
| Debian 12 arm64 | `$(OutDir)\runtimes\linux-arm64\liblzma.so` | Public Domain | glibc |
| macOS Big Sur x64 | `$(OutDir)\runtimes\osx-x64\liblzma.dylib` | Public Domain | libSystem |
| macOS Monterey arm64 | `$(OutDir)\runtimes\osx-arm64\liblzma.dylib` | Public Domain | libSystem |

- Bundled Windows binaires now target [Universal CRT](https://learn.microsoft.com/en-us/cpp/windows/universal-crt-deployment?view=msvc-170) for better interopability with MSVC.
- .NET Core/.NET 5+ runs on UCRT, so no action is required in most cases.
- If you encounter a dependency issue on Windows Vista, 7 or 8.1, try [installing UCRT manually](https://learn.microsoft.com/en-us/cpp/windows/universal-crt-deployment?view=msvc-170).
- If you call `XZInit.GlobalInit()` without `libPath` parameter on Linux or macOS, it will search for system-installed liblzma.
- If you call `XZInit.GlobalInit()` without the `libPath` parameter on Linux or macOS, it will search for system-installed liblzma.
- Linux binaries are not portable. They may not work on your distribution.
- You may call parameter-less `XZInit.GlobalInit()` to use system-installed liblzma.

Expand Down Expand Up @@ -166,7 +152,7 @@ You can tune xz compress options with this class.

| Property | Summary |
|----------|---------|
| Level | Compression level. The Default is `ZLibCompLevel.Default` (6). |
| Level | Compression level. The Default is `LzmaCompLevel.Default` (6). |
| ExtremeFlag | Use a slower variant to get a little bit better compression ratio hopefully. |
| BufferSize | Size of the internal buffer. The default is 1MB. |
| LeaveOpen | Whether to leave the base stream object open after disposing of the xz stream object. |
Expand Down Expand Up @@ -202,7 +188,7 @@ It may contain more advanced options.

**NOTE**: You must use threaded compression to let xz-utils decompress in parallel, even if you are using only 1 thread.

**WARNING**: If possible, always check the available system memory. Modern CPUs have a lot of cores, and each thread will allocate its buffer.
**WARNING**: If possible, always check the available system memory. Modern CPUs have a lot of cores, and each thread will allocate its own buffer.

**WARNING**: In multi-threaded compression, each thread may allocate more memory than the single-thread mode, including compressing in 1 thread. xz-utils aggressively buffers input and output in parallel compression. Use `XZInit.EncoderMemUsage()` to check the exact memory requirement for your config.

Expand Down Expand Up @@ -309,7 +295,7 @@ switch (XZInit.Lib.PlatformBitness)
{
case DynLoader.PlatformBitness.Bit32:
threadOpts.MemlimitThreading = Math.Min(XZHardware.PhysMem() / 4, 1400U << 20);
break;
break;s
case DynLoader.PlatformBitness.Bit64:
threadOpts.MemlimitThreading = XZHardware.PhysMem() / 4;
break;
Expand All @@ -327,11 +313,9 @@ using (XZStream zs = new XZStream(fsComp, decompOpts, threadOpts))

`Crc32Checksum` is the class designed to compute CRC32 checksum.

- Use `Append()` method to compute the checksum.
- Use `Checksum` property to get the checksum value.
- Use `Reset()` method to reset `Checksum` property.

**NOTE**: xz-utils provides about twice faster CRC32 implementation than zlib, in my benchmark with [Joveler.Compression.ZLib](https://www.nuget.org/packages/Joveler.Compression.ZLib).
- Use the `Append()` method to compute the checksum.
- Use the `Checksum` property to get the checksum value.
- Use the `Reset()` method to reset the `Checksum` property.

### Examples

Expand Down Expand Up @@ -379,9 +363,9 @@ It inherits and implements [HashAlgorithm](https://docs.microsoft.com/en-US/dotn

`Crc64Checksum` is the class designed to compute CRC64 checksum.

Use `Append()` method to compute the checksum.
Use `Checksum` property to get the checksum value.
Use `Reset()` method to reset `Checksum` property.
Use the `Append()` method to compute the checksum.
Use the `Checksum` property to get the checksum value.
Use the `Reset()` method to reset the `Checksum` property.

### Examples

Expand Down Expand Up @@ -420,6 +404,6 @@ using (FileStream fs = new FileStream("read.txt", FileMode.Open))

## Crc64Algorithm

`Crc64Algorithm` is the class designed to compute CRC64 checksum.
`Crc64Algorithm` is the class designed to compute the CRC64 checksum.

It inherits and implements [HashAlgorithm](https://docs.microsoft.com/en-US/dotnet/api/system.security.cryptography.hashalgorithm).
Binary file modified Joveler.Compression.XZ/runtimes/linux-arm/native/liblzma.so
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/linux-arm64/native/liblzma.so
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/linux-x64/native/liblzma.so
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/osx-arm64/native/liblzma.dylib
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/osx-x64/native/liblzma.dylib
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/win-arm64/native/liblzma.dll
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/win-x64/native/liblzma.dll
Binary file not shown.
Binary file modified Joveler.Compression.XZ/runtimes/win-x86/native/liblzma.dll
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
<PackageReference Include="MSTest.TestAdapter" Version="3.0.2" />
<PackageReference Include="MSTest.TestFramework" Version="3.0.2" />
<PackageReference Include="coverlet.collector" Version="3.2.0">
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.3" />
<PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
<PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
<PackageReference Include="coverlet.collector" Version="6.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
Expand Down
Loading

0 comments on commit 0b0b9c0

Please sign in to comment.