Merge pull request #270 from shipwright-io/fix/tar-size-calculation

Fix incorrect tar size calculation

pkg/shp/bundle/bundle.go

@@ -98,7 +98,10 @@ func Push(ctx context.Context, io *genericclioptions.IOStreams, localDirectory s
 					defer progress.Close()
 				}
 
-				progress.ChangeMax64(update.Total)
+				if update.Total != progress.GetMax64() {
+					progress.ChangeMax64(update.Total)
+				}
+
 				_ = progress.Set64(update.Complete)
 			}
 		}

pkg/shp/cmd/build/upload.go

@@ -210,14 +210,14 @@ func (u *UploadCommand) performDataStreaming(target *streamer.Target) error {
 		return err
 	}
 
-	tarsize, err := streamer.GetTarSize(u.sourceDir)
+	size, err := tarball.Size()
 	if err != nil {
 		return err
 	}
 
 	// start writing the data using the tarball format, and streaming it via STDIN, which is
 	// redirected to the correct container
-	if err = u.dataStreamer.Stream(target, tarball.Create, tarsize.Size); err != nil {
+	if err = u.dataStreamer.Stream(target, tarball.Create, size); err != nil {
 		return err
 	}
 

pkg/shp/streamer/streamer.go

@@ -9,10 +9,10 @@ import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-
-	progressbar "github.com/schollz/progressbar/v3"
 	"k8s.io/kubectl/pkg/cmd/exec"
 	"k8s.io/kubectl/pkg/util/interrupt"
+
+	progressbar "github.com/schollz/progressbar/v3"
 )
 
 // Streamer represents the actor that streams data onto a POD, running on Kubernetes. It does so via
@@ -43,7 +43,7 @@ func (s *Streamer) execute(opts *exec.ExecOptions) error {
 
 // Stream the data onto the informed target, and it uses the BaseDir as the path to store the data on
 // the running POD. The writerFn is employed to expose the writer interface to callers.
-func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int64) error {
+func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int) error {
 	var wg sync.WaitGroup
 	wg.Add(1)
 
@@ -60,7 +60,7 @@ func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int64) error {
 		wg.Done()
 	}()
 
-	progress := progressbar.NewOptions(int(size),
+	progress := progressbar.NewOptions(size,
 		progressbar.OptionSetWriter(os.Stderr),
 		progressbar.OptionEnableColorCodes(true),
 		progressbar.OptionShowBytes(true),

pkg/shp/streamer/streamer_test.go

@@ -38,11 +38,10 @@ func Test_Streamer(t *testing.T) {
 		BaseDir:   "/",
 	}
 
-	var size int64 = 1000
-
 	// streaming mocked standard input data, and asserting both command informed is expected, and
 	// stdin is preserved
 	stdin := "standard input"
+	size := len(stdin)
 	err := s.Stream(targetPod, func(w io.Writer) error {
 		_, err := w.Write([]byte(stdin))
 		return err

pkg/shp/streamer/tar.go

@@ -17,7 +17,6 @@ import (
 type Tar struct {
 	src       string            // base directory
 	gitIgnore *ignore.GitIgnore // matcher for git ignored files
-	Size      int64
 }
 
 // skipPath inspect each path and makes sure it skips files the tar helper can't handle.
@@ -69,38 +68,14 @@ func NewTar(src string) (*Tar, error) {
 	return t, t.bootstrap()
 }
 
-func GetTarSize(src string) (*Tar, error) {
-	t := &Tar{src: src}
-	return t, t.tarSize()
-}
-
-func (t *Tar) tarSize() error {
-	var size int64
-
-	err := filepath.WalkDir(t.src, func(fpath string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		stat, err := d.Info()
-		if err != nil {
-			return err
-		}
-
-		if t.skipPath(fpath, stat) {
-			return nil
-		}
-
-		header, err := tar.FileInfoHeader(stat, stat.Name())
-		if err != nil {
-			return err
-		}
-
-		header.Name = trimPrefix(t.src, fpath)
-		size += header.Size
-		return nil
-	})
+// Size returns the size in bytes that a call of Create would write into the io.Writer
+//
+// Note: This performs a whote walk through of the source directory and can be an expensive operation.
+func (t *Tar) Size() (int, error) {
+	wc := &writeCounter{}
+	if err := t.Create(wc); err != nil {
+		return -1, err
+	}
 
-	t.Size = size + size*1/100
-	return err
+	return wc.total, nil
 }

pkg/shp/streamer/util.go

@@ -9,6 +9,14 @@ import (
 	"strings"
 )
 
+type writeCounter struct{ total int }
+
+func (wc *writeCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.total += n
+	return n, nil
+}
+
 func trimPrefix(prefix, fpath string) string {
 	return strings.TrimPrefix(strings.Replace(fpath, prefix, "", -1), string(filepath.Separator))
 }