From 434718934f082471e5654998bb90510a024a8534 Mon Sep 17 00:00:00 2001 From: Krzysztof Kowalczyk Date: Mon, 4 Nov 2024 19:21:23 +0100 Subject: [PATCH] test various compression algos --- do/build.go | 21 +++++- do/go.mod | 1 + do/go.sum | 2 + do/main.go | 179 ++++++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 174 insertions(+), 29 deletions(-) diff --git a/do/build.go b/do/build.go index 3a9e9d38b34..63c5f4e8817 100644 --- a/do/build.go +++ b/do/build.go @@ -226,6 +226,17 @@ func revertBuildConfig() { runExeMust("git", "checkout", buildConfigPath()) } +func addZipDataStoreMust(w *zip.Writer, data []byte, nameInZip string) { + fih := &zip.FileHeader{ + Name: nameInZip, + Method: zip.Store, + } + fw, err := w.CreateHeader(fih) + must(err) + _, err = fw.Write(data) + must(err) +} + func addZipFileWithNameMust(w *zip.Writer, path, nameInZip string) { fi, err := os.Stat(path) must(err) @@ -308,9 +319,9 @@ func createPdbZipMust(dir string) { must(err) } -func createPdbLzsaMust(dir string) { - args := []string{"SumatraPDF.pdb.lzsa"} - args = append(args, pdbFiles...) +func createLzsaFromFiles(lzsaPath string, files []string, dir string) { + args := []string{lzsaPath} + args = append(args, files...) curDir, err := os.Getwd() must(err) makeLzsaPath := filepath.Join(curDir, "bin", "MakeLZSA.exe") @@ -319,6 +330,10 @@ func createPdbLzsaMust(dir string) { runCmdLoggedMust(cmd) } +func createPdbLzsaMust(dir string) { + createLzsaFromFiles("SumatraPDF.pdb.lzsa", pdbFiles, dir) +} + // manifest is build for pre-release builds and contains information about file sizes func createManifestMust() { var lines []string diff --git a/do/go.mod b/do/go.mod index 2434e5f87db..cad4e55b4b1 100644 --- a/do/go.mod +++ b/do/go.mod @@ -30,6 +30,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rs/xid v1.6.0 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect diff --git a/do/go.sum b/do/go.sum index ca97962efe1..aa81d0881c4 100644 --- a/do/go.sum +++ b/do/go.sum @@ -70,6 +70,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/do/main.go b/do/main.go index f902c07e39f..5ff36348f67 100644 --- a/do/main.go +++ b/do/main.go @@ -1,20 +1,27 @@ package main import ( + "archive/zip" + "bufio" + "bytes" "flag" "io" "os" "os/exec" "path/filepath" + "runtime" "strconv" "strings" + "sync" "time" + "github.com/andybalholm/brotli" "github.com/kjk/common/u" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz/lzma" ) var ( - flgSkipSign bool r2Access string r2Secret string b2Access string @@ -171,7 +178,6 @@ func runCppCheck(all bool) { } type BuildOptions struct { - sign bool upload bool verifyTranslationUpToDate bool doCleanCheck bool @@ -180,7 +186,6 @@ type BuildOptions struct { func ensureBuildOptionsPreRequesites(opts *BuildOptions) { logf("upload: %v\n", opts.upload) - logf("sign: %v\n", opts.sign) logf("verifyTranslationUpToDate: %v\n", opts.verifyTranslationUpToDate) if opts.upload { @@ -197,10 +202,6 @@ func ensureBuildOptionsPreRequesites(opts *BuildOptions) { verifyOnReleaseBranchMust() os.RemoveAll("out") } - - if !opts.sign { - flgSkipSign = true - } } func main() { @@ -370,17 +371,13 @@ func main() { detectVersions() if false { - testGenUpdateTxt() - return - } - - if false { - //buildPreRelease() - return - } - - if false { - deleteFilesOneOff() + testCompressOneOff() + if false { + // make them reachable + testGenUpdateTxt() + buildPreRelease(kPlatformIntel64, true) + deleteFilesOneOff() + } return } @@ -414,16 +411,8 @@ func main() { opts := &BuildOptions{} - if flgCIBuild { - // triggered via -ci from .github workflow file - // only sign if this is my repo (not a fork) - // master branch (not work branches) and on push (not pull requests etc.) - opts.sign = isGithubMyMasterBranch() - } - if flgUpload { // given by me from cmd-line - opts.sign = true opts.upload = true } @@ -610,3 +599,141 @@ func printBuildNoInfo(buildNo int) { s := lines[n] logf("%d: %s\n", buildNo, s) } + +func compressFileWithBrMust(path string) []byte { + buf := bytes.Buffer{} + w := brotli.NewWriterLevel(&buf, brotli.BestCompression) + f := u.Must2(os.Open(path)) + u.Must2(io.Copy(w, f)) + must(f.Close()) + must(w.Close()) + return buf.Bytes() +} + +func compressWithZstdMust(path string) []byte { + buf := bytes.Buffer{} + w := u.Must2(zstd.NewWriter(&buf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))) + f := u.Must2(os.Open(path)) + u.Must2(io.Copy(w, f)) + must(f.Close()) + must(w.Close()) + return buf.Bytes() +} + +func compressWithLzma2Must(path string) []byte { + buf := bytes.Buffer{} + bw := bufio.NewWriter(&buf) + w := u.Must2(lzma.NewWriter2(bw)) + f := u.Must2(os.Open(path)) + u.Must2(io.Copy(w, f)) + must(f.Close()) + must(w.Close()) + must(bw.Flush()) + return buf.Bytes() +} + +func compressWithLzma2BetterMust(path string) []byte { + buf := bytes.Buffer{} + bw := bufio.NewWriter(&buf) + var c lzma.Writer2Config + c.DictCap = (8 * 1024 * 1024) * 16 + must(c.Verify()) + w := u.Must2(c.NewWriter2(bw)) + f := u.Must2(os.Open(path)) + u.Must2(io.Copy(w, f)) + must(f.Close()) + must(w.Close()) + must(bw.Flush()) + return buf.Bytes() +} + +func creaZipWithCompressFunction(zipPath string, files []string, dir string, compressFunc func(string) []byte, comprSuffix string) { + os.Remove(zipPath) + w := u.Must2(os.Create(zipPath)) + zw := zip.NewWriter(w) + defer zw.Close() + var wg sync.WaitGroup + nConcurrent := runtime.NumCPU() + sem := make(chan bool, nConcurrent) + for _, f := range files { + path := filepath.Join(dir, f) + wg.Add(1) + sem <- true + go func() { + data := compressFunc(path) + addZipDataStoreMust(zw, data, f+comprSuffix) + <-sem + wg.Done() + }() + } + wg.Wait() + must(zw.Close()) +} + +func testCompressOneOff() { + dir := filepath.Join("out", "rel64") + files := []string{"SumatraPDF.exe", "SumatraPDF-dll.exe", "libmupdf.pdb", "SumatraPDF.pdb", "SumatraPDF-dll.pdb"} + { + } + origSize := int64(0) + for _, f := range files { + origSize += u.FileSize(filepath.Join(dir, f)) + } + logf("origSize: %s\n", u.FormatSize(origSize)) + + { + archivePath := filepath.Join(dir, "rel64.lzma2.better.zip") + os.Remove(archivePath) + logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU()) + printDur := measureDuration() + creaZipWithCompressFunction(archivePath, files, dir, compressWithLzma2BetterMust, ".lzma2") + printDur() + compressedSize := u.FileSize(archivePath) + ratio := float64(origSize) / float64(compressedSize) + logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio) + } + { + archivePath := filepath.Join(dir, "rel64.lzma2.zip") + os.Remove(archivePath) + logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU()) + printDur := measureDuration() + creaZipWithCompressFunction(archivePath, files, dir, compressWithLzma2Must, ".lzma2") + printDur() + compressedSize := u.FileSize(archivePath) + ratio := float64(origSize) / float64(compressedSize) + logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio) + } + { + archivePath := filepath.Join(dir, "rel64.br.zip") + os.Remove(archivePath) + logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU()) + printDur := measureDuration() + creaZipWithCompressFunction(archivePath, files, dir, compressFileWithBrMust, ".br") + printDur() + compressedSize := u.FileSize(archivePath) + ratio := float64(origSize) / float64(compressedSize) + logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio) + } + { + archivePath := filepath.Join(dir, "rel64.zstd.zip") + os.Remove(archivePath) + logf("\nCreating %s (%d threads)\n", archivePath, runtime.NumCPU()) + printDur := measureDuration() + creaZipWithCompressFunction(archivePath, files, dir, compressWithZstdMust, ".zstd") + printDur() + compressedSize := u.FileSize(archivePath) + ratio := float64(origSize) / float64(compressedSize) + logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio) + } + { + logf("\nCreating rel64.lzsa using\n") + printDur := measureDuration() + archivePath := filepath.Join(dir, "rel64.lzsa") + os.Remove(archivePath) + createLzsaFromFiles("rel64.lzsa", files, dir) + printDur() + compressedSize := u.FileSize(archivePath) + ratio := float64(origSize) / float64(compressedSize) + logf("compressedSize: %s, ratio: %.2f\n", u.FormatSize(compressedSize), ratio) + } +}