From a4d53a5da821455a6c1643bdce859655c3973883 Mon Sep 17 00:00:00 2001 From: Chris Gunn Date: Fri, 10 Jan 2025 12:18:37 -0800 Subject: [PATCH] Better handle filesystem optional features. (#66) Filesystems (e.g. ext4, xfs) sometimes receive new features. When these features are enabled, anything that loads the filesystem (e.g. kernel, grub, etc.) must support the new feature or it will refuse to load it. When formatting a new partition, there are a few considerations for if a filesystem feature should be enabled: - Does the version of mkfs support that feature? - Does the build host kernel support that feature? - Does the target OS support that feature? This change ensure that all these considerations are handled correctly for the ext4 and xfs filesystem types. --- toolkit/tools/imagegen/diskutils/diskutils.go | 31 +- .../tools/imagegen/diskutils/filesystem.go | 324 ++++++++++++++++++ toolkit/tools/imager/imager.go | 5 +- toolkit/tools/internal/envfile/envfile.go | 80 +++++ .../internal/kernelversion/kernelversion.go | 59 ++++ .../internal/safemount/safemount_test.go | 3 +- toolkit/tools/internal/targetos/targetos.go | 51 +++ toolkit/tools/internal/version/version.go | 69 ++++ .../tools/internal/version/version_test.go | 46 +++ .../customizepartitionsfilecopy.go | 8 +- .../pkg/imagecustomizerlib/imageutils.go | 24 +- .../imagecustomizerlib/liveosisobuilder.go | 9 +- 12 files changed, 676 insertions(+), 33 deletions(-) create mode 100644 toolkit/tools/imagegen/diskutils/filesystem.go create mode 100644 toolkit/tools/internal/envfile/envfile.go create mode 100644 toolkit/tools/internal/kernelversion/kernelversion.go create mode 100644 toolkit/tools/internal/targetos/targetos.go create mode 100644 toolkit/tools/internal/version/version.go create mode 100644 toolkit/tools/internal/version/version_test.go diff --git a/toolkit/tools/imagegen/diskutils/diskutils.go b/toolkit/tools/imagegen/diskutils/diskutils.go index c8c9f18704..175ff78e8d 100644 --- a/toolkit/tools/imagegen/diskutils/diskutils.go +++ b/toolkit/tools/imagegen/diskutils/diskutils.go @@ -20,21 +20,11 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/retry" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" "github.com/sirupsen/logrus" ) var ( - // When calling mkfs, the default options change depending on the host OS you are running on and typically match - // what the distro has decided is best for their OS. For example, for ext2/3/4, the defaults are stored in - // /etc/mke2fs.conf. - // However, when building Azure Linux images, the defaults should be as consistent as possible and should only contain - // features that are supported on Azure Linux. - DefaultMkfsOptions = map[string][]string{ - "ext2": {"-b", "4096", "-O", "none,sparse_super,large_file,filetype,resize_inode,dir_index,ext_attr"}, - "ext3": {"-b", "4096", "-O", "none,sparse_super,large_file,filetype,resize_inode,dir_index,ext_attr,has_journal"}, - "ext4": {"-b", "4096", "-O", "none,sparse_super,large_file,filetype,resize_inode,dir_index,ext_attr,has_journal,extent,huge_file,flex_bg,metadata_csum,64bit,dir_nlink,extra_isize"}, - } - partedVersionRegex = regexp.MustCompile(`^parted \(GNU parted\) (\d+)\.(\d+)`) // The default partition name used when the version of `parted` is too old (<3.5). @@ -433,7 +423,7 @@ func WaitForDevicesToSettle() error { } // CreatePartitions creates partitions on the specified disk according to the disk config -func CreatePartitions(diskDevPath string, disk configuration.Disk, rootEncryption configuration.RootEncryption, +func CreatePartitions(targetOs targetos.TargetOs, diskDevPath string, disk configuration.Disk, rootEncryption configuration.RootEncryption, diskKnownToBeEmpty bool, ) (partDevPathMap map[string]string, partIDToFsTypeMap map[string]string, encryptedRoot EncryptedRootDevice, err error) { const timeoutInSeconds = "5" @@ -492,7 +482,7 @@ func CreatePartitions(diskDevPath string, disk configuration.Disk, rootEncryptio return partDevPathMap, partIDToFsTypeMap, encryptedRoot, err } - partFsType, err := FormatSinglePartition(partDevPath, partition) + partFsType, err := formatSinglePartition(targetOs, partDevPath, partition) if err != nil { err = fmt.Errorf("failed to format partition:\n%w", err) return partDevPathMap, partIDToFsTypeMap, encryptedRoot, err @@ -789,8 +779,8 @@ func setGptPartitionType(partition configuration.Partition, timeoutInSeconds, di return } -// FormatSinglePartition formats the given partition to the type specified in the partition configuration -func FormatSinglePartition(partDevPath string, partition configuration.Partition, +// formatSinglePartition formats the given partition to the type specified in the partition configuration +func formatSinglePartition(targetOs targetos.TargetOs, partDevPath string, partition configuration.Partition, ) (fsType string, err error) { const ( totalAttempts = 5 @@ -804,12 +794,17 @@ func FormatSinglePartition(partDevPath string, partition configuration.Partition // To handle such cases, we can retry the command. switch fsType { case "fat32", "fat16", "vfat", "ext2", "ext3", "ext4", "xfs": - mkfsOptions := DefaultMkfsOptions[fsType] - if fsType == "fat32" || fsType == "fat16" { fsType = "vfat" } + mkfsOptions, err := getFileSystemOptions(targetOs, fsType) + if err != nil { + err = fmt.Errorf("failed to get mkfs args for filesystem type (%s) and target os (%s):\n%w", fsType, + targetOs, err) + return fsType, err + } + mkfsArgs := []string{"-t", fsType} mkfsArgs = append(mkfsArgs, mkfsOptions...) mkfsArgs = append(mkfsArgs, partDevPath) @@ -825,6 +820,7 @@ func FormatSinglePartition(partDevPath string, partition configuration.Partition }, totalAttempts, retryDuration) if err != nil { err = fmt.Errorf("could not format partition with type %v after %v retries", fsType, totalAttempts) + return "", err } case "linux-swap": err = retry.Run(func() error { @@ -837,6 +833,7 @@ func FormatSinglePartition(partDevPath string, partition configuration.Partition }, totalAttempts, retryDuration) if err != nil { err = fmt.Errorf("could not format partition with type %v after %v retries", fsType, totalAttempts) + return "", err } _, stderr, err := shell.Execute("swapon", partDevPath) diff --git a/toolkit/tools/imagegen/diskutils/filesystem.go b/toolkit/tools/imagegen/diskutils/filesystem.go new file mode 100644 index 0000000000..fd010b81e5 --- /dev/null +++ b/toolkit/tools/imagegen/diskutils/filesystem.go @@ -0,0 +1,324 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package diskutils + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/kernelversion" + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" + "github.com/microsoft/azurelinux/toolkit/tools/internal/version" +) + +// When calling mkfs, the default options change depending on the host OS you are running on and typically match +// what the distro has decided is best for their OS. For example, for ext2/3/4, the defaults are stored in +// /etc/mke2fs.conf. +// However, when building Azure Linux images, the defaults should be as consistent as possible and should only contain +// features that are supported on Azure Linux. + +type ext4Options struct { + BlockSize int + Features []string +} + +type xfsOptions struct { + Features []string +} + +type fileSystemsOptions struct { + Ext4 ext4Options + Xfs xfsOptions +} + +var ( + // The default ext4 options used by an Azure Linux 2.0 image. + // See, the /etc/mke2fs.conf file in an Azure Linux 2.0 image. + azl2Ext4Options = ext4Options{ + BlockSize: 4096, + Features: []string{"sparse_super", "large_file", "filetype", "resize_inode", "dir_index", "ext_attr", + "has_journal", "extent", "huge_file", "flex_bg", "metadata_csum", "64bit", "dir_nlink", "extra_isize", + }, + } + + // The default ext4 options used by an Azure Linux 3.0 image. + // See, the /etc/mke2fs.conf file in an Azure Linux 3.0 image. + azl3Ext4Options = ext4Options{ + BlockSize: 4096, + Features: []string{"sparse_super", "large_file", "filetype", "resize_inode", "dir_index", "ext_attr", + "has_journal", "extent", "huge_file", "flex_bg", "metadata_csum", "64bit", "dir_nlink", "extra_isize", + "orphan_file", + }, + } + + // The default xfs options used by an Azure Linux 2.0 image (kernel v5.15). + // See, the /usr/share/xfsprogs/mkfs/lts_5.15.conf file. + azl2XfsOptions = xfsOptions{ + Features: []string{"bigtime", "crc", "finobt", "inobtcount", "reflink", "rmapbt", "sparse"}, + } + + // The default xfs options used by an Azure Linux 3.0 image (kernel v6.6) + // See, the /usr/share/xfsprogs/mkfs/lts_6.6.conf file. + azl3XfsOptions = xfsOptions{ + Features: []string{"bigtime", "crc", "finobt", "inobtcount", "reflink", "rmapbt", "sparse", "nrext64"}, + } + + targetOsFileSystemsOptions = map[targetos.TargetOs]fileSystemsOptions{ + targetos.TargetOsAzureLinux2: { + Ext4: azl2Ext4Options, + Xfs: azl2XfsOptions, + }, + targetos.TargetOsAzureLinux3: { + Ext4: azl3Ext4Options, + Xfs: azl3XfsOptions, + }, + } + + // A list of ext4 features and their minimum supported Linux kernel version. + // + // Note: This list omits features that either: + // - Are not used by one of the supported distros/versions, OR + // - Are supported by MinKernelVersion (v5.4). + // + // Ref: https://www.man7.org/linux/man-pages/man5/ext4.5.html + ext4FeaturesKernelSupport = map[string]version.Version{ + "orphan_file": {5, 15}, + } + + // A list of ext4 features and their minimum supported e2fsprogs versions. + // + // Note: This list omits features that either: + // - Are not used by one of the supported distros/versions, OR + // - Are supported by MinKernelVersion (v5.4). + // + // Ref: https://e2fsprogs.sourceforge.net/e2fsprogs-release.html + ext4FeaturesE2fsprogsSupport = map[string]version.Version{ + "orphan_file": {1, 47, 0}, + } + + // A list of XFS features and their minimum supported xfsprogs / kernel versions. + // + // Note: XFS tools are developed in the Linux kernel tree. Hence, the kernel and xfsprogs versions are tied + // together. + xfsFeaturesSupport = map[string]version.Version{ + "bigtime": {5, 10}, + "crc": {3, 2, 0}, + "finobt": {3, 2, 1}, + "inobtcount": {5, 10}, + "reflink": {4, 10}, + "rmapbt": {4, 10}, + "sparse": {4, 10}, + "nrext64": {5, 19}, + } + + // The mkfs.xfs flag each feature sits under. + xfsFeatureFlag = map[string]string{ + "bigtime": "metadata", + "crc": "metadata", + "finobt": "metadata", + "inobtcount": "metadata", + "reflink": "metadata", + "rmapbt": "metadata", + "sparse": "inode", + "nrext64": "inode", + } + + // The maximum version of mkfs.xfs that is currently supported. + // This is used to prevent issues with newer versions of mkfs.xfs default enabling new features. + maxMkfsXfsVersion = version.Version{6, 9} + + // The minimum supported kernel version. This helps avoid versions complexity for features that are old and therefore + // basically universal. + // + // Relevant kernel versions: + // - Ubuntu 22.04: v5.15 + // - Mariner 2.0: v5.15 + // - Mariner 3.0: v6.6 + minKernelVersion = version.Version{5, 4} + + // For exampke: mke2fs 1.47.0 (5-Feb-2023) + mke2fsVersionRegex = regexp.MustCompile(`(?m)^mke2fs (\d+)\.(\d+)\.(\d+) \(\d+-[a-zA-Z]+-\d+\)$`) + + // For example: mkfs.xfs version 6.5.0 + mkfsXfsVersionRegex = regexp.MustCompile(`^mkfs\.xfs version (\d+)\.(\d+)\.(\d+)$`) +) + +func getFileSystemOptions(targetOs targetos.TargetOs, filesystemType string) ([]string, error) { + hostKernelVersion, err := kernelversion.GetBuildHostKernelVersion() + if err != nil { + return nil, fmt.Errorf("failed to get host kernel version:\n%w", err) + } + + if minKernelVersion.Gt(hostKernelVersion) { + return nil, fmt.Errorf("host kernel version (%s) is too old (min: %s)", minKernelVersion, hostKernelVersion) + } + + options, hasOptions := targetOsFileSystemsOptions[targetOs] + if !hasOptions { + return nil, fmt.Errorf("unknown target OS (%s)", targetOs) + } + + switch filesystemType { + case "ext4": + options, err := getExt4FileSystemOptions(hostKernelVersion, options) + if err != nil { + return nil, err + } + + return options, nil + + case "xfs": + options, err := getXfsFileSystemOptions(hostKernelVersion, options) + if err != nil { + return nil, err + } + + return options, nil + + default: + return []string(nil), nil + } +} + +func getExt4FileSystemOptions(hostKernelVersion version.Version, options fileSystemsOptions) ([]string, error) { + mke2fsVersion, err := getMke2fsVersion() + if err != nil { + return nil, err + } + + // "none" requests no default options. + features := []string{"none"} + + for _, feature := range options.Ext4.Features { + requiredKernelVersion, hasRequiredKernelVersion := ext4FeaturesKernelSupport[feature] + if hasRequiredKernelVersion && requiredKernelVersion.Gt(hostKernelVersion) { + // Feature is not supported on build host kernel. + logger.Log.Infof("Build host kernel does not support ext4 feature (%s)", feature) + continue + } + + requiredMke2fsVersion, hasRequiredMke2fsVersion := ext4FeaturesE2fsprogsSupport[feature] + if hasRequiredMke2fsVersion && requiredMke2fsVersion.Gt(mke2fsVersion) { + // Feature is not supported by version of mkfs.ext4. + logger.Log.Infof("mkfs.ext4 does not support ext4 feature (%s)", feature) + continue + } + + features = append(features, feature) + } + + featuresArg := strings.Join(features, ",") + + args := []string{"-b", strconv.Itoa(options.Ext4.BlockSize), "-O", featuresArg} + return args, nil +} + +func getXfsFileSystemOptions(hostKernelVersion version.Version, options fileSystemsOptions) ([]string, error) { + mkfsXfsVersion, err := getMkfsXfsVersion() + if err != nil { + return nil, err + } + + if mkfsXfsVersion.Gt(maxMkfsXfsVersion) { + // New versions of mkfs.xfs might add new default-enabled features in the future. + // So, block newer versions of mkfs.xfs until we have verified there aren't any new XFS features that need to + // set in the CLI args. + return nil, fmt.Errorf("mkfs.xfs version (%s) is too new (max: %s)", mkfsXfsVersion, maxMkfsXfsVersion) + } + + metadataArgs := []string(nil) + inodeArgs := []string(nil) + + // Unlike mkfs.ext4, mkfs.xfs doesn't have a mechanism to disable all features. + // So, explictly set every feature flag. + for feature, requiredVersion := range xfsFeaturesSupport { + enableFeature := sliceutils.ContainsValue(options.Xfs.Features, feature) + + if requiredVersion.Gt(hostKernelVersion) { + // Feature is not supported on build host kernel. + if enableFeature { + logger.Log.Infof("Build host kernel does not support xfs feature (%s)", feature) + enableFeature = false + } + } + + if requiredVersion.Gt(mkfsXfsVersion) { + // Feature is not supported by mkfs.xfs. + if enableFeature { + logger.Log.Infof("mkfs.xfs does not support xfs feature (%s)", feature) + } + + // This version of mkfs.xfs will not recognize the CLI option. + // So, don't include it. + continue + } + + enableArg := "0" + if enableFeature { + enableArg = "1" + } + featureArg := fmt.Sprintf("%s=%s", feature, enableArg) + + switch xfsFeatureFlag[feature] { + case "metadata": + metadataArgs = append(metadataArgs, featureArg) + + case "inode": + inodeArgs = append(inodeArgs, featureArg) + } + } + + metadataArgValue := strings.Join(metadataArgs, ",") + inodeArgValue := strings.Join(inodeArgs, ",") + + args := []string{"-m", metadataArgValue, "-i", inodeArgValue} + return args, nil +} + +// Get the version of mkfs.ext4 +func getMke2fsVersion() (version.Version, error) { + _, stderr, err := shell.Execute("mke2fs", "-V") + if err != nil { + return nil, fmt.Errorf("failed to get mke2fs's version:\n%w", err) + } + + fullVersionString := strings.TrimSpace(stderr) + + match := mke2fsVersionRegex.FindStringSubmatch(fullVersionString) + if match == nil { + return nil, fmt.Errorf("failed to parse mke2fs's version (%s)", fullVersionString) + } + + major, _ := strconv.Atoi(match[1]) + minor, _ := strconv.Atoi(match[2]) + patch, _ := strconv.Atoi(match[3]) + version := version.Version{major, minor, patch} + return version, nil +} + +// Get the version of mkfs.xfs +func getMkfsXfsVersion() (version.Version, error) { + stdout, _, err := shell.Execute("mkfs.xfs", "-V") + if err != nil { + return nil, fmt.Errorf("failed to get mkfs.xfs's version:\n%w", err) + } + + fullVersionString := strings.TrimSpace(stdout) + + match := mkfsXfsVersionRegex.FindStringSubmatch(fullVersionString) + if match == nil { + return nil, fmt.Errorf("failed to parse mkfs.xfs's version (%s)", fullVersionString) + } + + major, _ := strconv.Atoi(match[1]) + minor, _ := strconv.Atoi(match[2]) + patch, _ := strconv.Atoi(match[3]) + version := version.Version{major, minor, patch} + return version, nil +} diff --git a/toolkit/tools/imager/imager.go b/toolkit/tools/imager/imager.go index 80b937a770..96e6f05f6a 100644 --- a/toolkit/tools/imager/imager.go +++ b/toolkit/tools/imager/imager.go @@ -19,6 +19,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" "github.com/microsoft/azurelinux/toolkit/tools/internal/tdnf" "github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp" "github.com/microsoft/azurelinux/toolkit/tools/pkg/profile" @@ -422,8 +423,8 @@ func setupLoopDeviceDisk(outputDir, diskName string, diskConfig configuration.Di func setupRealDisk(diskDevPath string, diskConfig configuration.Disk, rootEncryption configuration.RootEncryption, diskKnownToBeEmpty bool, ) (partIDToDevPathMap, partIDToFsTypeMap map[string]string, encryptedRoot diskutils.EncryptedRootDevice, err error) { // Set up partitions - partIDToDevPathMap, partIDToFsTypeMap, encryptedRoot, err = diskutils.CreatePartitions(diskDevPath, diskConfig, - rootEncryption, diskKnownToBeEmpty) + partIDToDevPathMap, partIDToFsTypeMap, encryptedRoot, err = diskutils.CreatePartitions( + targetos.TargetOsAzureLinux3, diskDevPath, diskConfig, rootEncryption, diskKnownToBeEmpty) if err != nil { err = fmt.Errorf("failed to create partitions on disk (%s):\n%w", diskDevPath, err) return diff --git a/toolkit/tools/internal/envfile/envfile.go b/toolkit/tools/internal/envfile/envfile.go new file mode 100644 index 0000000000..70b90e87fa --- /dev/null +++ b/toolkit/tools/internal/envfile/envfile.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Used to parse config files formatted like a Bash script file containing only variable assignments. + +package envfile + +import ( + "fmt" + "strings" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/file" + "github.com/microsoft/azurelinux/toolkit/tools/internal/grub" +) + +func ParseEnvFile(path string) (map[string]string, error) { + content, err := file.Read(path) + if err != nil { + return nil, err + } + + return ParseEnv(content) +} + +func ParseEnv(content string) (map[string]string, error) { + tokens, err := grub.TokenizeConfig(content) + if err != nil { + return nil, err + } + + result := make(map[string]string) + + lines := grub.SplitTokensIntoLines(tokens) + for _, line := range lines { + if len(line.Tokens) > 2 { + loc := line.Tokens[1].Loc.Start + return nil, fmt.Errorf("env file line has multiple words (%d:%d)", loc.Line, loc.Col) + } + + token := line.Tokens[0] + + // Variable assignments can not have any character escaping before the '=' char. + if token.Type != grub.WORD && + token.SubWords[0].Type != grub.KEYWORD_STRING { + loc := token.Loc.Start + return nil, fmt.Errorf("env file line is not a variable assignment (%d:%d)", loc.Line, loc.Col) + } + + firstWord := token.SubWords[0].Value + + // Find the '=' char. + eqIndex := strings.Index(firstWord, "=") + if eqIndex < 0 { + loc := token.Loc.Start + return nil, fmt.Errorf("env file line is not a variable assignment (%d:%d)", loc.Line, loc.Col) + } + + name := firstWord[:eqIndex] + + valueBuilder := strings.Builder{} + valueBuilder.WriteString(firstWord[eqIndex+1:]) + + for _, word := range token.SubWords[1:] { + switch word.Type { + case grub.KEYWORD_STRING, grub.STRING: + valueBuilder.WriteString(word.Value) + + default: + loc := word.Loc.Start + return nil, fmt.Errorf("env file contains invalid characters (%d:%d)", loc.Line, loc.Col) + } + } + + value := valueBuilder.String() + + result[name] = value + } + + return result, nil +} diff --git a/toolkit/tools/internal/kernelversion/kernelversion.go b/toolkit/tools/internal/kernelversion/kernelversion.go new file mode 100644 index 0000000000..8fc37fd8f4 --- /dev/null +++ b/toolkit/tools/internal/kernelversion/kernelversion.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kernelversion + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/version" + "golang.org/x/sys/unix" +) + +var ( + // Parses the kernel version from "uname -r" or subdirectories of /lib/modules. + // + // Examples: + // OS Version + // Fedora 40 6.11.6-200.fc40.x86_64 + // Ubuntu 22.04 6.8.0-48-generic + // Azure Linux 2.0 5.15.153.1-2.cm2 + // Azure Linux 3.0 6.6.47.1-1.azl3 + kernelVersionRegex = regexp.MustCompile(`^(\d+)\.(\d+)\.(\d+)([.\-][a-zA-Z0-9_.\-]*)?$`) +) + +func GetBuildHostKernelVersion() (version.Version, error) { + utsName := unix.Utsname{} + err := unix.Uname(&utsName) + if err != nil { + return nil, fmt.Errorf("failed to query uname:\n%w", err) + } + + versionBuf := utsName.Release[:] + versionStringLen := bytes.IndexByte(versionBuf, 0) + versionString := string(versionBuf[:versionStringLen]) + + version, err := parseKernelVersion(versionString) + if err != nil { + return nil, err + } + + return version, nil +} + +func parseKernelVersion(versionString string) (version.Version, error) { + match := kernelVersionRegex.FindStringSubmatch(versionString) + if match == nil { + return nil, fmt.Errorf("failed to parse kernel version (%s)", versionString) + } + + major, _ := strconv.Atoi(match[1]) + minor, _ := strconv.Atoi(match[2]) + patch, _ := strconv.Atoi(match[3]) + + version := version.Version{major, minor, patch} + return version, nil +} diff --git a/toolkit/tools/internal/safemount/safemount_test.go b/toolkit/tools/internal/safemount/safemount_test.go index b6ae872293..9fb13df2c5 100644 --- a/toolkit/tools/internal/safemount/safemount_test.go +++ b/toolkit/tools/internal/safemount/safemount_test.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/buildpipeline" "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/safeloopback" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" "github.com/moby/sys/mountinfo" "github.com/stretchr/testify/assert" ) @@ -66,7 +67,7 @@ func TestResourceBusy(t *testing.T) { defer loopback.Close() // Set up partitions. - _, _, _, err = diskutils.CreatePartitions(loopback.DevicePath(), diskConfig, + _, _, _, err = diskutils.CreatePartitions(targetos.TargetOsAzureLinux3, loopback.DevicePath(), diskConfig, configuration.RootEncryption{}, true /*diskKnownToBeEmpty*/) if !assert.NoError(t, err, "failed to create partitions on disk", loopback.DevicePath()) { return diff --git a/toolkit/tools/internal/targetos/targetos.go b/toolkit/tools/internal/targetos/targetos.go new file mode 100644 index 0000000000..5b67c6053a --- /dev/null +++ b/toolkit/tools/internal/targetos/targetos.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package targetos + +import ( + "fmt" + "path/filepath" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/envfile" +) + +type TargetOs string + +const ( + TargetOsAzureLinux2 TargetOs = "azl2" + TargetOsAzureLinux3 TargetOs = "azl3" +) + +func GetInstalledTargetOs(rootfs string) (TargetOs, error) { + fields, err := envfile.ParseEnvFile(filepath.Join(rootfs, "etc/os-release")) + if err != nil { + return "", fmt.Errorf("failed to read /etc/os-release file:\n%w", err) + } + + distroId := fields["ID"] + versionId := fields["VERSION_ID"] + + switch distroId { + case "mariner": + switch versionId { + case "2.0": + return TargetOsAzureLinux2, nil + + default: + return "", fmt.Errorf("unknown VERSION_ID (%s) for CBL-Mariner in /etc/os-release", versionId) + } + + case "azurelinux": + switch versionId { + case "3.0": + return TargetOsAzureLinux3, nil + + default: + return "", fmt.Errorf("unknown VERSION_ID (%s) for Azure Linux in /etc/os-release", versionId) + } + + default: + return "", fmt.Errorf("unknown ID (%s) in /etc/os-release", distroId) + } +} diff --git a/toolkit/tools/internal/version/version.go b/toolkit/tools/internal/version/version.go new file mode 100644 index 0000000000..5f9e8a5d7b --- /dev/null +++ b/toolkit/tools/internal/version/version.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package version + +import ( + "fmt" + "strings" +) + +type Version []int + +func (v Version) Cmp(other Version) int { + count := len(v) + if len(other) > count { + count = len(other) + } + + for i := 0; i < count; i++ { + c1 := 0 + if i < len(v) { + c1 = v[i] + } + + c2 := 0 + if i < len(other) { + c2 = other[i] + } + + if c1 > c2 { + return 1 + } else if c1 < c2 { + return -1 + } + } + + return 0 +} + +func (v Version) Gt(other Version) bool { + return v.Cmp(other) > 0 +} + +func (v Version) Ge(other Version) bool { + return v.Cmp(other) >= 0 +} + +func (v Version) Lt(other Version) bool { + return v.Cmp(other) < 0 +} + +func (v Version) Le(other Version) bool { + return v.Cmp(other) <= 0 +} + +func (v Version) Eq(other Version) bool { + return v.Cmp(other) == 0 +} + +func (v Version) String() string { + builder := strings.Builder{} + for i, p := range v { + if i != 0 { + builder.WriteString(".") + } + builder.WriteString(fmt.Sprintf("%d", p)) + } + return builder.String() +} diff --git a/toolkit/tools/internal/version/version_test.go b/toolkit/tools/internal/version/version_test.go new file mode 100644 index 0000000000..dd0d607fc6 --- /dev/null +++ b/toolkit/tools/internal/version/version_test.go @@ -0,0 +1,46 @@ +// Copyright Microsoft Corporation. +// Licensed under the MIT License. + +package version + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVersionGt(t *testing.T) { + assert.True(t, Version{2}.Gt(Version{1})) + assert.True(t, Version{2}.Gt(Version{1, 0})) + assert.True(t, Version{2}.Gt(Version{1, 1})) + assert.True(t, Version{2, 0}.Gt(Version{1})) + assert.True(t, Version{2, 1}.Gt(Version{1})) +} + +func TestVersionGe(t *testing.T) { + assert.True(t, Version{2}.Ge(Version{1})) + assert.True(t, Version{2}.Ge(Version{2})) +} + +func TestVersionLt(t *testing.T) { + assert.True(t, Version{1}.Lt(Version{2})) +} + +func TestVersionLe(t *testing.T) { + assert.True(t, Version{1}.Le(Version{2})) + assert.True(t, Version{1}.Le(Version{1})) +} + +func TestVersionEq(t *testing.T) { + assert.True(t, Version{1}.Eq(Version{1})) + assert.True(t, Version{1}.Eq(Version{1, 0})) + assert.True(t, Version{1, 0}.Eq(Version{1})) + assert.True(t, Version{1, 0}.Eq(Version{1, 0})) +} + +func TestVersionString(t *testing.T) { + assert.Equal(t, "", Version{}.String()) + assert.Equal(t, "1", Version{1}.String()) + assert.Equal(t, "1.2", Version{1, 2}.String()) + assert.Equal(t, "1.2.3", Version{1, 2, 3}.String()) +} diff --git a/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go b/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go index f4121a5386..740f388ba2 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go +++ b/toolkit/tools/pkg/imagecustomizerlib/customizepartitionsfilecopy.go @@ -9,6 +9,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/imagecustomizerapi" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" "github.com/sirupsen/logrus" ) @@ -21,13 +22,18 @@ func customizePartitionsUsingFileCopy(buildDir string, baseConfigPath string, co } defer existingImageConnection.Close() + targetOs, err := targetos.GetInstalledTargetOs(existingImageConnection.Chroot().RootDir()) + if err != nil { + return nil, fmt.Errorf("failed to determine target OS of base image:\n%w", err) + } + diskConfig := config.Storage.Disks[0] installOSFunc := func(imageChroot *safechroot.Chroot) error { return copyFilesIntoNewDisk(existingImageConnection.Chroot(), imageChroot) } - partIdToPartUuid, err := createNewImage(newBuildImageFile, diskConfig, config.Storage.FileSystems, + partIdToPartUuid, err := createNewImage(targetOs, newBuildImageFile, diskConfig, config.Storage.FileSystems, buildDir, "newimageroot", installOSFunc) if err != nil { return nil, err diff --git a/toolkit/tools/pkg/imagecustomizerlib/imageutils.go b/toolkit/tools/pkg/imagecustomizerlib/imageutils.go index 03a58c8578..08d6f8ac42 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/imageutils.go +++ b/toolkit/tools/pkg/imagecustomizerlib/imageutils.go @@ -15,6 +15,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" "github.com/microsoft/azurelinux/toolkit/tools/internal/sliceutils" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" ) type installOSFunc func(imageChroot *safechroot.Chroot) error @@ -57,15 +58,15 @@ func connectToExistingImageHelper(imageConnection *ImageConnection, imageFilePat return nil } -func createNewImage(filename string, diskConfig imagecustomizerapi.Disk, +func createNewImage(targetOs targetos.TargetOs, filename string, diskConfig imagecustomizerapi.Disk, fileSystems []imagecustomizerapi.FileSystem, buildDir string, chrootDirName string, installOS installOSFunc, ) (map[string]string, error) { imageConnection := NewImageConnection() defer imageConnection.Close() - partIdToPartUuid, err := createNewImageHelper(imageConnection, filename, diskConfig, fileSystems, buildDir, chrootDirName, - installOS) + partIdToPartUuid, err := createNewImageHelper(targetOs, imageConnection, filename, diskConfig, fileSystems, + buildDir, chrootDirName, installOS) if err != nil { return nil, fmt.Errorf("failed to create new image:\n%w", err) } @@ -79,9 +80,9 @@ func createNewImage(filename string, diskConfig imagecustomizerapi.Disk, return partIdToPartUuid, nil } -func createNewImageHelper(imageConnection *ImageConnection, filename string, diskConfig imagecustomizerapi.Disk, - fileSystems []imagecustomizerapi.FileSystem, buildDir string, chrootDirName string, - installOS installOSFunc, +func createNewImageHelper(targetOs targetos.TargetOs, imageConnection *ImageConnection, filename string, + diskConfig imagecustomizerapi.Disk, fileSystems []imagecustomizerapi.FileSystem, buildDir string, + chrootDirName string, installOS installOSFunc, ) (map[string]string, error) { // Convert config to image config types, so that the imager's utils can be used. @@ -96,8 +97,8 @@ func createNewImageHelper(imageConnection *ImageConnection, filename string, dis } // Create imager boilerplate. - partIdToPartUuid, tmpFstabFile, err := createImageBoilerplate(imageConnection, filename, buildDir, chrootDirName, - imagerDiskConfig, imagerPartitionSettings) + partIdToPartUuid, tmpFstabFile, err := createImageBoilerplate(targetOs, imageConnection, filename, buildDir, + chrootDirName, imagerDiskConfig, imagerPartitionSettings) if err != nil { return nil, err } @@ -160,8 +161,9 @@ func configureDiskBootLoader(imageConnection *ImageConnection, rootMountIdType i return nil } -func createImageBoilerplate(imageConnection *ImageConnection, filename string, buildDir string, chrootDirName string, - imagerDiskConfig configuration.Disk, imagerPartitionSettings []configuration.PartitionSetting, +func createImageBoilerplate(targetOs targetos.TargetOs, imageConnection *ImageConnection, filename string, + buildDir string, chrootDirName string, imagerDiskConfig configuration.Disk, + imagerPartitionSettings []configuration.PartitionSetting, ) (map[string]string, string, error) { // Create raw disk image file. err := diskutils.CreateSparseDisk(filename, imagerDiskConfig.MaxSize, 0o644) @@ -177,7 +179,7 @@ func createImageBoilerplate(imageConnection *ImageConnection, filename string, b // Set up partitions. partIDToDevPathMap, partIDToFsTypeMap, _, err := diskutils.CreatePartitions( - imageConnection.Loopback().DevicePath(), imagerDiskConfig, configuration.RootEncryption{}, + targetOs, imageConnection.Loopback().DevicePath(), imagerDiskConfig, configuration.RootEncryption{}, true /*diskKnownToBeEmpty*/) if err != nil { return nil, "", fmt.Errorf("failed to create partitions on disk (%s):\n%w", imageConnection.Loopback().DevicePath(), err) diff --git a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go index 045e4c495f..92da32b877 100644 --- a/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go +++ b/toolkit/tools/pkg/imagecustomizerlib/liveosisobuilder.go @@ -22,6 +22,7 @@ import ( "github.com/microsoft/azurelinux/toolkit/tools/internal/safeloopback" "github.com/microsoft/azurelinux/toolkit/tools/internal/safemount" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" + "github.com/microsoft/azurelinux/toolkit/tools/internal/targetos" "github.com/microsoft/azurelinux/toolkit/tools/pkg/isomakerlib" "golang.org/x/sys/unix" ) @@ -1969,6 +1970,11 @@ func (b *LiveOSIsoBuilder) createWriteableImageFromSquashfs(buildDir, rawImageFi }, } + targetOs, err := targetos.GetInstalledTargetOs(squashMountDir) + if err != nil { + return fmt.Errorf("failed to determine target OS of ISO squashfs:\n%w", err) + } + // populate the newly created disk image with content from the squash fs installOSFunc := func(imageChroot *safechroot.Chroot) error { // At the point when this copy will be executed, both the boot and the @@ -1984,7 +1990,8 @@ func (b *LiveOSIsoBuilder) createWriteableImageFromSquashfs(buildDir, rawImageFi // create the new raw disk image writeableChrootDir := "writeable-raw-image" - _, err = createNewImage(rawImageFile, diskConfig, fileSystemConfigs, buildDir, writeableChrootDir, installOSFunc) + _, err = createNewImage(targetOs, rawImageFile, diskConfig, fileSystemConfigs, buildDir, writeableChrootDir, + installOSFunc) if err != nil { return fmt.Errorf("failed to copy squashfs into new writeable image (%s):\n%w", rawImageFile, err) }