disk: fix sfdisk for in-use disk

We are getting the error now:
  This disk is currently in use - repartitioning is probably a bad idea.
  Umount all file systems, and swapoff all swap partitions on this disk.
  Use the --no-reread flag to suppress this check.

Fixes: 1191e24

build/gather-node-deps.sh

@@ -6,7 +6,7 @@ mkdir -p /staging-node/var/lib/dpkg/status.d
 
 DEPS=(
     /etc/mke2fs.conf /sbin/{fsck,mkfs,mount,umount}.{ext{2,3,4},xfs,nfs}
-    /usr/bin/{mount,umount,lspci,lsof,chmod,grep,tail,nsenter}
+    /usr/bin/{mount,umount,lspci,lsof,chmod,grep,tail,nsenter,partx}
     /usr/sbin/{fsck,mkfs,sfdisk,losetup,blockdev}
     /sbin/dumpe2fs /sbin/resize2fs
     /usr/sbin/xfs_io /usr/sbin/xfs_growfs

build/multi/Dockerfile.multi

@@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-cache-$TARGET
     echo 'Acquire::Check-Valid-Until false;' > /etc/apt/apt.conf.d/snapshot && \
     sed -i '/^URIs:/d; s|^# \(http://snapshot.debian.org/\)|URIs: \1|' /etc/apt/sources.list.d/debian.sources && \
     apt-get update && \
-    apt-get install -y nfs-common e2fsprogs xfsprogs pciutils fdisk lsof
+    apt-get install -y nfs-common e2fsprogs xfsprogs pciutils fdisk util-linux lsof
 
 RUN --mount=type=bind,from=distroless-base,target=/base \
     --mount=type=bind,source=build/gather-node-deps.sh,target=/deps.sh \

pkg/disk/sfdisk/expand.go

@@ -8,46 +8,35 @@ import (
 
 	utilsos "github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/utils/os"
 
-	"golang.org/x/sys/unix"
 	"k8s.io/klog/v2"
 )
 
 func ExpandPartition(ctx context.Context, disk, partition string) error {
 	logger := klog.FromContext(ctx)
-	fd, err := unix.Open(disk, unix.O_RDONLY, 0)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := unix.Close(fd); err != nil {
-			logger.Error(err, "failed to close", "fd", fd)
-		}
-	}()
-
-	err = unix.Flock(fd, unix.LOCK_EX) // as suggested in the man sfdisk(8)
-	if err != nil {
-		return fmt.Errorf("failed to lock %s exclusively: %v", disk, err)
-	}
-	defer func() {
-		if err := unix.Flock(fd, unix.LOCK_UN); err != nil {
-			logger.Error(err, "failed to unlock", "fd", fd)
-		}
-	}()
 
 	dump, err := exec.CommandContext(ctx, "sfdisk", "--dump", disk).Output()
 	if err != nil {
-		return fmt.Errorf("failed to dump current partition table of %s: %v", disk, utilsos.ErrWithStderr(err))
+		return fmt.Errorf("failed to dump current partition table of %s: %w", disk, utilsos.ErrWithStderr(err))
 	}
 	dumpStr := string(dump)
 	logger.V(4).Info("sfdisk dump before expansion", "dump", dumpStr)
 
 	// Don't cancel this, we don't want to corrupt the partition table
-	cmd := exec.Command("sfdisk", disk, "-N", partition)
+	// --lock according to https://systemd.io/BLOCK_DEVICE_LOCKING/
+	// --no-reread --no-tell-kernel is necessary for online expansion. We will use partx to update the kernel.
+	cmd := exec.Command("sfdisk", "--lock", "--no-reread", "--no-tell-kernel", disk, "-N", partition)
 	cmd.Stdin = bytes.NewReader([]byte(",+")) // enlarge the partition as much as possible
 	result, err := cmd.Output()
 	if err != nil {
-		return fmt.Errorf("failed to expand partition %s on %s: %v\noriginal table looked like:\n%s", partition, disk, utilsos.ErrWithStderr(err), dumpStr)
+		return fmt.Errorf("failed to expand partition %s on %s: %w\noriginal table looked like:\n%s", partition, disk, utilsos.ErrWithStderr(err), dumpStr)
 	}
 	logger.V(3).Info("sfdisk success", "output", string(result))
+
+	cmd = exec.Command("partx", "--update", disk, "--nr", partition)
+	result, err = cmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to update partition %s on %s: %w", partition, disk, utilsos.ErrWithStderr(err))
+	}
+	logger.V(3).Info("partx success", "output", string(result))
 	return nil
 }

pkg/disk/sfdisk/expand_test.go

@@ -6,36 +6,70 @@ import (
 	"errors"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"testing"
 
 	utilsos "github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/utils/os"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func runCommand(t *testing.T, name string, arg ...string) string {
+	cmd := exec.Command(name, arg...)
+	output, err := cmd.Output()
+	require.NoError(t, utilsos.ErrWithStderr(err))
+	return string(output)
+}
+
 func TestExpandPartition(t *testing.T) {
 	path, err := exec.LookPath("sfdisk")
 	if errors.Is(err, exec.ErrNotFound) {
 		t.Skip("sfdisk not found")
 	}
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	t.Logf("sfdisk found at: %s", path)
 
 	testImage := t.TempDir() + "/test.img"
-	_, err = os.Create(testImage)
-	assert.NoError(t, err)
-	assert.NoError(t, os.Truncate(testImage, 1<<23)) // 8MB
+	f, err := os.Create(testImage)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	require.NoError(t, os.Truncate(testImage, 1<<23)) // 8MB
 
 	cmd := exec.Command("sfdisk", testImage)
 	cmd.Stdin = bytes.NewReader([]byte("label: gpt\n,\n")) // create a single partition
 	result, err := cmd.Output()
-	assert.NoError(t, utilsos.ErrWithStderr(err))
+	require.NoError(t, utilsos.ErrWithStderr(err))
 	t.Logf("create partition success: %s", string(result))
 
-	assert.NoError(t, os.Truncate(testImage, 1<<24)) // expand to 16MB
-	assert.NoError(t, ExpandPartition(context.Background(), testImage, "1"))
+	result, err = exec.Command("losetup", "--find", testImage, "--show", "--partscan").Output()
+	if err != nil {
+		t.Skip("losetup failed", err)
+	}
+	disk := strings.TrimSpace(string(result))
+	t.Logf("loop device: %s", disk)
+	defer func() {
+		runCommand(t, "losetup", "-d", disk)
+	}()
 
-	dump, err := exec.Command("sfdisk", "--dump", testImage).Output()
-	assert.NoError(t, utilsos.ErrWithStderr(err))
-	assert.Contains(t, strings.ReplaceAll(string(dump), " ", ""), "test.img1:start=2048,size=30687")
+	part := disk + "p1"
+	output := runCommand(t, "mkfs.ext4", "-F", part)
+	t.Logf("mkfs.ext4 output: %s", output)
+	mntPath := t.TempDir()
+	runCommand(t, "mount", part, mntPath)
+	defer func() {
+		runCommand(t, "umount", mntPath)
+	}()
+
+	require.NoError(t, os.Truncate(testImage, 1<<24)) // expand to 16MB
+	runCommand(t, "losetup", "--set-capacity", disk)
+	require.NoError(t, ExpandPartition(context.Background(), disk, "1"))
+
+	dump := runCommand(t, "sfdisk", "--dump", disk)
+	assert.Contains(t, strings.ReplaceAll(dump, " ", ""), "p1:start=2048,size=30687")
+
+	output = runCommand(t, "blockdev", "--getsize64", part)
+	realSize, err := strconv.Atoi(strings.TrimSpace(output))
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, realSize, 12<<20)
 }