Skip to content

Commit

Permalink
Merge pull request #16 from gary-lgy/generic-e2e-resources
Browse files Browse the repository at this point in the history
  • Loading branch information
@gary-lgy
gary-lgy authored Apr 15, 2023
2 parents 80049b0 + 3269154 commit 4fcf830
Show file tree
Hide file tree
Showing 6 changed files with 445 additions and 322 deletions.
26 changes: 26 additions & 0 deletions test/e2e/framework/resources/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/utils/pointer"

"github.com/kubewharf/kubeadmiral/pkg/controllers/common"
)

var DefaultPodImage = "ealen/echo-server:latest"
Expand All @@ -47,6 +49,10 @@ func GetSimpleDeployment(baseName string) *appsv1.Deployment {
name := fmt.Sprintf("%s-%s", baseName, rand.String(12))

return &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
APIVersion: appsv1.SchemeGroupVersion.String(),
Kind: common.DeploymentKind,
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Expand Down Expand Up @@ -76,6 +82,18 @@ func GetSimpleDeployment(baseName string) *appsv1.Deployment {
}
}

func IsDeploymentProgressing(deployment *appsv1.Deployment) bool {
for _, c := range deployment.Status.Conditions {
if c.Type != appsv1.DeploymentProgressing {
continue
}

return c.Status == corev1.ConditionTrue
}

return false
}

func GetSimpleDaemonset() *appsv1.DaemonSet {
return nil
}
Expand All @@ -88,6 +106,10 @@ func GetSimpleJob(baseName string) *batchv1.Job {
name := fmt.Sprintf("%s-%s", baseName, rand.String(12))

return &batchv1.Job{
TypeMeta: metav1.TypeMeta{
APIVersion: batchv1.SchemeGroupVersion.String(),
Kind: common.JobKind,
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Expand Down Expand Up @@ -121,6 +143,10 @@ func GetSimpleCronJob(baseName string) *batchv1b1.CronJob {
name := fmt.Sprintf("%s-%s", baseName, rand.String(12))

return &batchv1b1.CronJob{
TypeMeta: metav1.TypeMeta{
APIVersion: batchv1.SchemeGroupVersion.String(),
Kind: common.CronJobKind,
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/framework/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ func AssertForItems[T any](
ginkgo.Fail(buf.String())
}

// PollUntilForItems polls each item in items with condFn until
// it returns true, an error, or until ctx is cancelled.
func PollUntilForItems[T any](
ctx context.Context,
items []T,
Expand Down
164 changes: 22 additions & 142 deletions test/e2e/resourcepropagation/cronjobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,158 +17,38 @@ limitations under the License.
package resourcepropagation

import (
"context"
"time"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
batchv1b1 "k8s.io/api/batch/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"

fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1"
fedtypesv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/types/v1alpha1"
"github.com/kubewharf/kubeadmiral/test/e2e/framework"
"github.com/kubewharf/kubeadmiral/test/e2e/framework/policies"
"github.com/kubewharf/kubeadmiral/test/e2e/framework/resources"
"github.com/kubewharf/kubeadmiral/test/e2e/framework/util"
)

var _ = ginkgo.Describe("CronJob Propagation", resourcePropagationTestLabel, func() {
var _ = ginkgo.Describe("CronJob Propagation", func() {
f := framework.NewFramework("cronjob-propagation", framework.FrameworkOptions{CreateNamespace: true})

var cronJob *batchv1b1.CronJob
var clusters []*fedcorev1a1.FederatedCluster

ginkgo.BeforeEach(func(ctx ginkgo.SpecContext) {
cronJob = resources.GetSimpleCronJob(f.Name())
clusterList, err := f.HostFedClient().CoreV1alpha1().FederatedClusters().List(ctx, metav1.ListOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)

clusters = make([]*fedcorev1a1.FederatedCluster, len(clusterList.Items))
for i := range clusterList.Items {
clusters[i] = &clusterList.Items[i]
}
clusters = util.FilterOutE2ETestObjects(clusters)

policy := policies.PropagationPolicyForClustersWithPlacements(f.Name(), clusters)
policies.SetPropagationPolicy(cronJob, policy)

_, err = f.HostFedClient().CoreV1alpha1().PropagationPolicies(f.TestNamespace().Name).Create(
ctx,
policy,
metav1.CreateOptions{},
)
gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
})

assertCronJobsPropagated := func(ctx context.Context) {
ctxWithTimeout, cancel := context.WithTimeout(ctx, resourcePropagationTimeout)
defer cancel()

failedClusters, err := util.PollUntilForItems(
ctxWithTimeout,
clusters,
func(c *fedcorev1a1.FederatedCluster) (bool, error) {
_, err := f.ClusterKubeClient(ctx, c).BatchV1beta1().CronJobs(f.TestNamespace().Name).Get(
ctx,
cronJob.Name,
metav1.GetOptions{},
)
if err != nil && apierrors.IsNotFound(err) {
return false, nil
}
return true, err
resourcePropagationTest(
f,
&resourcePropagationTestConfig[*batchv1b1.CronJob]{
gvr: batchv1b1.SchemeGroupVersion.WithResource("jobs"),
objectFactory: resources.GetSimpleCronJob,
clientGetter: func(client kubernetes.Interface, namespace string) resourceClient[*batchv1b1.CronJob] {
return client.BatchV1beta1().CronJobs(namespace)
},
defaultPollingInterval,
)

gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
gomega.Expect(failedClusters).
To(gomega.BeEmpty(), "Timed out waiting for cronjob to propagate to clusters %v", util.NameList(failedClusters))
}

assertCronJobsRunOnce := func(ctx context.Context) {
ctxWithTimeout, cancel := context.WithTimeout(ctx, resourceReadyTimeout)
defer cancel()

failedClusters, err := util.PollUntilForItems(
ctxWithTimeout,
clusters,
func(c *fedcorev1a1.FederatedCluster) (bool, error) {
clusterJob, err := f.ClusterKubeClient(ctx, c).BatchV1beta1().CronJobs(f.TestNamespace().Name).Get(
ctx,
cronJob.Name,
metav1.GetOptions{},
)
if err != nil {
return true, err
}
return resources.IsCronJobScheduledOnce(clusterJob), nil
isPropagatedResourceWorking: func(
_ kubernetes.Interface,
_ dynamic.Interface,
cronjob *batchv1b1.CronJob,
) (bool, error) {
return resources.IsCronJobScheduledOnce(cronjob), nil
},
defaultPollingInterval,
)

gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
gomega.Expect(failedClusters).
To(gomega.BeEmpty(), "Timed out waiting for cronjob %s to be run once in clusters %v", cronJob.Name, failedClusters)
}

assertCronJobsDeleted := func(ctx context.Context) {
ctxWithTimeout, cancel := context.WithTimeout(ctx, resourceDeleteTimeout)
defer cancel()

failedClusters, err := util.PollUntilForItems(
ctxWithTimeout,
clusters,
func(c *fedcorev1a1.FederatedCluster) (bool, error) {
_, err := f.ClusterKubeClient(ctx, c).BatchV1beta1().CronJobs(f.TestNamespace().Name).Get(
ctx,
cronJob.Name,
metav1.GetOptions{},
)
if err != nil && !apierrors.IsNotFound(err) {
return true, err
}
return apierrors.IsNotFound(err), nil
statusCollection: &resourceStatusCollectionTestConfig{
gvr: fedtypesv1a1.SchemeGroupVersion.WithResource("federatedcronjobstatuses"),
path: "status",
},
defaultPollingInterval,
)
gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
gomega.Expect(failedClusters).
To(gomega.BeEmpty(), "Timed out waiting for job to be deleted in clusters %v", failedClusters)

gomega.Eventually(func(g gomega.Gomega, ctx context.Context) {
_, err := f.HostKubeClient().BatchV1beta1().CronJobs(f.TestNamespace().Name).Get(ctx, cronJob.Name, metav1.GetOptions{})
gomega.Expect(err).To(gomega.Or(gomega.BeNil(), gomega.Satisfy(apierrors.IsNotFound)))
g.Expect(err).To(gomega.Satisfy(apierrors.IsNotFound))
}).WithContext(ctxWithTimeout).Should(gomega.Succeed(), "Timed out waiting for source object deletion")
}

ginkgo.It("should succeed", func(ctx ginkgo.SpecContext) {
var err error

ginkgo.By("Creating cronjob")
cronJob, err = f.HostKubeClient().BatchV1beta1().CronJobs(f.TestNamespace().Name).Create(ctx, cronJob, metav1.CreateOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
ginkgo.GinkgoLogr.Info("created cronJob for propagation", "cronjob-name", cronJob.Name, "cronjob-namespace", f.TestNamespace().Name)

start := time.Now()

ginkgo.By("Waiting for cronjob propagation")
assertCronJobsPropagated(ctx)
ginkgo.GinkgoLogr.Info("all cronjobs propagated", "duration", time.Since(start))

ginkgo.By("Waiting for cronjob to be scheduled once")
assertCronJobsRunOnce(ctx)
ginkgo.GinkgoLogr.Info("all cronjobs run once", "duration", time.Since(start))

err = f.HostKubeClient().BatchV1beta1().CronJobs(f.TestNamespace().Name).Delete(ctx, cronJob.Name, metav1.DeleteOptions{})
gomega.Expect(err).ToNot(gomega.HaveOccurred(), framework.MessageUnexpectedError)
ginkgo.GinkgoLogr.Info("deleted source cronjob object")

start = time.Now()
ginkgo.By("Waiting for cronjob deletion")
assertCronJobsDeleted(ctx)
ginkgo.GinkgoLogr.Info("all cronjobs deleted", "duration", time.Since(start))
})
},
)
})
54 changes: 54 additions & 0 deletions test/e2e/resourcepropagation/deployments.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
Copyright 2023 The KubeAdmiral Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resourcepropagation

import (
"github.com/onsi/ginkgo/v2"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"

fedtypesv1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/types/v1alpha1"
"github.com/kubewharf/kubeadmiral/test/e2e/framework"
"github.com/kubewharf/kubeadmiral/test/e2e/framework/resources"
)

var _ = ginkgo.Describe("Deployment Propagation", func() {
f := framework.NewFramework("deployment-propagation", framework.FrameworkOptions{CreateNamespace: true})

resourcePropagationTest(
f,
&resourcePropagationTestConfig[*appsv1.Deployment]{
gvr: appsv1.SchemeGroupVersion.WithResource("deployments"),
objectFactory: resources.GetSimpleDeployment,
clientGetter: func(client kubernetes.Interface, namespace string) resourceClient[*appsv1.Deployment] {
return client.AppsV1().Deployments(namespace)
},
isPropagatedResourceWorking: func(
_ kubernetes.Interface,
_ dynamic.Interface,
deployment *appsv1.Deployment,
) (bool, error) {
return resources.IsDeploymentProgressing(deployment), nil
},
statusCollection: &resourceStatusCollectionTestConfig{
gvr: fedtypesv1a1.SchemeGroupVersion.WithResource("federateddeploymentstatuses"),
path: "status",
},
},
)
})
Loading

0 comments on commit 4fcf830

Please sign in to comment.