feat(occm): support multi region cluster

Currently, it supports only single auth section. Set the regions in config as: [Global] region=REGION1 regions=REGION1 regions=REGION2 regions=REGION3
kubernetes · Dec 20, 2024 · dcd162b · dcd162b
1 parent bbb82f4
commit dcd162b
Show file tree

Hide file tree

Showing 6 changed files with 175 additions and 60 deletions.
diff --git a/.../openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md b/.../openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md
@@ -113,7 +113,9 @@ The options in `Global` section are used for openstack-cloud-controller-manager
 * `password`
   Keystone user password. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required.
 * `region`
-  Required. Keystone region name.
+  Required. Keystone region name. The name of region will be set in the `topology.kubernetes.io/region` label of the node.
+* `regions`
+  Optional. Keystone region name, which is used to specify regions for the cloud provider where the instance is running. Can be specified multiple times.The region name may or may not be the same as the region name in the `region` option. They merge together at runtime.
 * `domain-id`
   Keystone user domain ID. If you are using [Keystone application credential](https://docs.openstack.org/keystone/latest/user/application_credentials.html), this option is not required.
 * `domain-name`
@@ -317,7 +319,7 @@ Although the openstack-cloud-controller-manager was initially implemented with N
   call](https://docs.openstack.org/api-ref/load-balancer/v2/?expanded=create-a-load-balancer-detail#creating-a-fully-populated-load-balancer).
   Setting this option to true will create loadbalancers using serial API calls which first create an unpopulated
   loadbalancer, then populate its listeners, pools and members. This is a compatibility option at the expense of
-  increased load on the OpenStack API. Default: false 
+  increased load on the OpenStack API. Default: false
 
 NOTE:
 

diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"slices"
 	"strings"
 
 	"github.com/gophercloud/gophercloud/v2"
@@ -53,6 +54,7 @@ type AuthOpts struct {
 	UserDomainID     string                   `gcfg:"user-domain-id" mapstructure:"user-domain-id" name:"os-userDomainID" value:"optional"`
 	UserDomainName   string                   `gcfg:"user-domain-name" mapstructure:"user-domain-name" name:"os-userDomainName" value:"optional"`
 	Region           string                   `name:"os-region"`
+	Regions          []string                 `name:"os-regions" value:"optional"`
 	EndpointType     gophercloud.Availability `gcfg:"os-endpoint-type" mapstructure:"os-endpoint-type" name:"os-endpointType" value:"optional"`
 	CAFile           string                   `gcfg:"ca-file" mapstructure:"ca-file" name:"os-certAuthorityPath" value:"optional"`
 	TLSInsecure      string                   `gcfg:"tls-insecure" mapstructure:"tls-insecure" name:"os-TLSInsecure" value:"optional" matches:"^true|false$"`
@@ -87,6 +89,7 @@ func LogCfg(authOpts AuthOpts) {
 	klog.V(5).Infof("UserDomainID: %s", authOpts.UserDomainID)
 	klog.V(5).Infof("UserDomainName: %s", authOpts.UserDomainName)
 	klog.V(5).Infof("Region: %s", authOpts.Region)
+	klog.V(5).Infof("Regions: %s", authOpts.Regions)
 	klog.V(5).Infof("EndpointType: %s", authOpts.EndpointType)
 	klog.V(5).Infof("CAFile: %s", authOpts.CAFile)
 	klog.V(5).Infof("CertFile: %s", authOpts.CertFile)
@@ -232,6 +235,20 @@ func ReadClouds(authOpts *AuthOpts) error {
 	authOpts.ApplicationCredentialName = replaceEmpty(authOpts.ApplicationCredentialName, cloud.AuthInfo.ApplicationCredentialName)
 	authOpts.ApplicationCredentialSecret = replaceEmpty(authOpts.ApplicationCredentialSecret, cloud.AuthInfo.ApplicationCredentialSecret)
 
+	regions := strings.Split(authOpts.Region, ",")
+	if len(regions) > 1 {
+		authOpts.Region = regions[0]
+	}
+
+	for _, r := range cloud.Regions {
+		// Support only single auth section in clouds.yaml
+		if r.Values.AuthInfo == nil && r.Name != authOpts.Region && !slices.Contains(regions, r.Name) {
+			regions = append(regions, r.Name)
+		}
+	}
+
+	authOpts.Regions = regions
+
 	return nil
 }
 

diff --git a/pkg/csi/cinder/openstack/openstack.go b/pkg/csi/cinder/openstack/openstack.go
@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"slices"
 
 	"github.com/gophercloud/gophercloud/v2"
 	"github.com/gophercloud/gophercloud/v2/openstack"
@@ -126,7 +127,7 @@ func GetConfigFromFiles(configFilePaths []string) (Config, error) {
 		}
 	}
 
-	for _, global := range cfg.Global {
+	for idx, global := range cfg.Global {
 		// Update the config with data from clouds.yaml if UseClouds is enabled
 		if global.UseClouds {
 			if global.CloudsFile != "" {
@@ -138,6 +139,15 @@ func GetConfigFromFiles(configFilePaths []string) (Config, error) {
 			}
 			klog.V(5).Infof("Credentials are loaded from %s:", global.CloudsFile)
 		}
+
+		regions := []string{global.Region}
+		for _, region := range cfg.Global[idx].Regions {
+			if !slices.Contains(regions, region) {
+				regions = append(regions, region)
+			}
+		}
+
+		cfg.Global[idx].Regions = regions
 	}
 
 	return cfg, nil

diff --git a/pkg/csi/cinder/openstack/openstack_test.go b/pkg/csi/cinder/openstack/openstack_test.go
@@ -68,6 +68,7 @@ tenant-id=` + fakeTenantID + `
 domain-id=` + fakeDomainID + `
 ca-file=` + fakeCAfile + `
 region=` + fakeRegion + `
+regions=` + fakeRegion + `
 [Global "cloud2"]
 username=` + fakeUserName_cloud2 + `
 password=` + fakePassword_cloud2 + `
@@ -76,6 +77,8 @@ tenant-id=` + fakeTenantID_cloud2 + `
 domain-id=` + fakeDomainID_cloud2 + `
 ca-file=` + fakeCAfile_cloud2 + `
 region=` + fakeRegion_cloud2 + `
+regions=` + fakeRegion_cloud2 + `
+regions=` + fakeRegion_cloud2 + `
 [Global "cloud3"]
 username=` + fakeUserName_cloud3 + `
 password=` + fakePassword_cloud3 + `
@@ -112,6 +115,7 @@ rescan-on-resize=true`
 		CAFile:   fakeCAfile,
 		TenantID: fakeTenantID,
 		Region:   fakeRegion,
+		Regions:  []string{fakeRegion},
 	}
 	expectedOpts.Global["cloud2"] = &client.AuthOpts{
 		Username: fakeUserName_cloud2,
@@ -121,6 +125,7 @@ rescan-on-resize=true`
 		CAFile:   fakeCAfile_cloud2,
 		TenantID: fakeTenantID_cloud2,
 		Region:   fakeRegion_cloud2,
+		Regions:  []string{fakeRegion_cloud2},
 	}
 	expectedOpts.Global["cloud3"] = &client.AuthOpts{
 		Username: fakeUserName_cloud3,
@@ -130,6 +135,7 @@ rescan-on-resize=true`
 		CAFile:   fakeCAfile_cloud3,
 		TenantID: fakeTenantID_cloud3,
 		Region:   fakeRegion_cloud3,
+		Regions:  []string{fakeRegion_cloud3},
 	}
 
 	expectedOpts.BlockStorage.RescanOnResize = true
@@ -224,6 +230,7 @@ rescan-on-resize=true`
 		CAFile:       fakeCAfile,
 		TenantID:     fakeTenantID,
 		Region:       fakeRegion,
+		Regions:      []string{fakeRegion},
 		EndpointType: gophercloud.AvailabilityPublic,
 		UseClouds:    true,
 		CloudsFile:   wd + "/fixtures/clouds.yaml",
@@ -237,6 +244,7 @@ rescan-on-resize=true`
 		CAFile:       fakeCAfile_cloud2,
 		TenantID:     fakeTenantID_cloud2,
 		Region:       fakeRegion_cloud2,
+		Regions:      []string{fakeRegion_cloud2},
 		EndpointType: gophercloud.AvailabilityPublic,
 		UseClouds:    true,
 		CloudsFile:   wd + "/fixtures/clouds.yaml",
@@ -250,6 +258,7 @@ rescan-on-resize=true`
 		CAFile:       fakeCAfile_cloud3,
 		TenantID:     fakeTenantID_cloud3,
 		Region:       fakeRegion_cloud3,
+		Regions:      []string{fakeRegion_cloud3},
 		EndpointType: gophercloud.AvailabilityPublic,
 		UseClouds:    true,
 		CloudsFile:   wd + "/fixtures/clouds.yaml",