From b70ef6400b448429c870f2afc82572cf3d07f742 Mon Sep 17 00:00:00 2001 From: kzadorozhny Date: Fri, 25 Jun 2021 14:43:00 -0700 Subject: [PATCH 1/2] updating cluster setup and e2e script --- create_kind_cluster.sh | 16 ++++++++++++---- delete_kind_cluster.sh | 27 ++++++++++++++++++++++++++ examples/e2e-test.sh | 43 +++++++++++++++++++++++++++++------------- 3 files changed, 69 insertions(+), 17 deletions(-) create mode 100755 delete_kind_cluster.sh diff --git a/create_kind_cluster.sh b/create_kind_cluster.sh index 5acdc161..3d2e59d3 100755 --- a/create_kind_cluster.sh +++ b/create_kind_cluster.sh @@ -10,6 +10,7 @@ # governing permissions and limitations under the License. set -o errexit + # desired cluster name; default is "kind" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" @@ -19,19 +20,26 @@ reg_port='5000' running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" if [ "${running}" != 'true' ]; then docker run \ - -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ registry:2 fi -reg_ip="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" # create a cluster with the local registry enabled in containerd -cat < Date: Fri, 25 Jun 2021 17:50:32 -0700 Subject: [PATCH 2/2] WIP --- examples/.bazelrc | 3 ++ examples/WORKSPACE | 17 +++++++++ examples/helloworld/BUILD | 26 +++++++++++++- .../helloworld/helloworld_integration_test.go | 34 ++++++++++++++++++ skylib/k8s.bzl | 35 ++++++++++++++----- 5 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 examples/helloworld/helloworld_integration_test.go diff --git a/examples/.bazelrc b/examples/.bazelrc index 2cd9942b..4af64c9b 100644 --- a/examples/.bazelrc +++ b/examples/.bazelrc @@ -1,2 +1,5 @@ +# print out tests output in case of error +test --test_output=errors + # required for rules_docker 0.15 common --incompatible_restrict_string_escapes=false diff --git a/examples/WORKSPACE b/examples/WORKSPACE index 396bc6a1..02f2b110 100644 --- a/examples/WORKSPACE +++ b/examples/WORKSPACE @@ -32,3 +32,20 @@ load( ) go_image_repositories() + +# this is required to load the kubeconfig repository_rule +# http_archive( +# name = "io_bazel_rules_k8s", +# sha256 = "cc75cf0d86312e1327d226e980efd3599704e01099b58b3c2fc4efe5e321fcd9", +# strip_prefix = "rules_k8s-0.3.1", +# urls = ["https://github.com/bazelbuild/rules_k8s/releases/download/v0.3.1/rules_k8s-v0.3.1.tar.gz"], +# ) + +load("@com_adobe_rules_gitops//skylib:k8s.bzl", "kubeconfig") + +kubeconfig( + name = "k8s_test", + cluster = "kind-kind", + # server = "https://127.0.0.1:62816", + # symlink = True, +) diff --git a/examples/helloworld/BUILD b/examples/helloworld/BUILD index b297deda..b30287b4 100644 --- a/examples/helloworld/BUILD +++ b/examples/helloworld/BUILD @@ -12,7 +12,7 @@ licenses(["notice"]) # Apache 2.0 load("@io_bazel_rules_docker//go:image.bzl", "go_image") load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") -load("@com_adobe_rules_gitops//gitops:defs.bzl", "k8s_deploy") +load("@com_adobe_rules_gitops//gitops:defs.bzl", "k8s_deploy", "k8s_test_setup") go_library( name = "go_default_library", @@ -138,3 +138,27 @@ sh_test( "@bazel_tools//tools/bash/runfiles", ], ) + +k8s_test_setup( + name = "service_it.setup", + kubeconfig = "@k8s_test//:kubeconfig", + objects = [ + ":mynamespace", + ], +) + +go_test( + name = "helloworld_integration_test", + srcs = ["helloworld_integration_test.go"], + args = [ + "-setup", + "$(location :service_it.setup)", + ], + data = [ + ":service_it.setup", + ], + rundir = ".", + deps = [ + "@com_adobe_rules_gitops//testing/it_sidecar/client:go_default_library", + ], +) diff --git a/examples/helloworld/helloworld_integration_test.go b/examples/helloworld/helloworld_integration_test.go new file mode 100644 index 00000000..3b914bed --- /dev/null +++ b/examples/helloworld/helloworld_integration_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "net/http" + "io/ioutil" + "strings" + "testing" + + "github.com/adobe/rules_gitops/testing/it_sidecar/client" +) + +var setup client.K8STestSetup + +func TestMain(m *testing.M) { + setup = client.K8STestSetup{PortForwardServices: map[string]int{"helloworld": 8080}} + setup.TestMain(m) +} + +func TestSuccessfulReceival(t *testing.T) { + localPort := setup.GetServiceLocalPort("helloworld") + fmt.Printf("helloworld server is available at localport %d\n", localPort) + resp, err := http.Get(fmt.Sprintf("http://localhost:%d", localPort)) + if err != nil { + t.Fatalf("request error %s", err) + } + if resp.StatusCode != 200 { + t.Fatalf("Unexpected status code %d, expectted 200", resp.StatusCode) + } + body, _ := ioutil.ReadAll(resp.Body) + if !strings.Contains(string(body), "Hello World") { + t.Error("Unexpected content returned:", string(body)) + } +} diff --git a/skylib/k8s.bzl b/skylib/k8s.bzl index a29ddf39..f3875b72 100644 --- a/skylib/k8s.bzl +++ b/skylib/k8s.bzl @@ -340,10 +340,27 @@ def _kubeconfig_impl(repository_ctx): kubecert_cert = certs.get_child("kubecert.cert") kubecert_key = certs.get_child("kubecert.key") + # detect server endpoint + # Note: this is an ugly hack + if not server: + exec_result = repository_ctx.execute(["kubectl", "cluster-info", "--cluster=%s" % repository_ctx.attr.cluster]) + if exec_result.return_code != 0: + fail("Error detecting %s cluster server endpoint" % repository_ctx.attr.cluster) + cluster_info = exec_result.stdout.splitlines() + server_start_index = cluster_info[0].find("https://") + if server_start_index >= 0: + # remove traling terminal escape sequence + server_part = cluster_info[0][server_start_index:] + server = "" + for i in range(len(server_part)): + c = server_part[i] + if not (c.isalnum() or c in "/:."): + break + server += c + # config set-cluster {cluster} \ # --certificate-authority=... \ # --server=https://dev3.k8s.tubemogul.info:443 \ - # --embed-certs", _kubectl_config(repository_ctx, [ "set-cluster", repository_ctx.attr.cluster, @@ -353,7 +370,7 @@ def _kubeconfig_impl(repository_ctx): ca_crt, ]) - # config set-credentials {user} --token=...", + # config set-credentials {user} --token=... if token: _kubectl_config(repository_ctx, [ "set-credentials", @@ -362,7 +379,7 @@ def _kubeconfig_impl(repository_ctx): token, ]) - # config set-credentials {user} --client-certificate=... --embed-certs", + # config set-credentials {user} --client-certificate=... if kubecert_cert and kubecert_cert.exists: _kubectl_config(repository_ctx, [ "set-credentials", @@ -371,7 +388,7 @@ def _kubeconfig_impl(repository_ctx): kubecert_cert.realpath, ]) - # config set-credentials {user} --client-key=... --embed-certs", + # config set-credentials {user} --client-key=... if kubecert_key and kubecert_key.exists: _kubectl_config(repository_ctx, [ "set-credentials", @@ -430,7 +447,7 @@ def _k8s_cmd_impl(ctx): command_file = ctx.actions.declare_file(ctx.label.name + ".command") _stamp(ctx, ctx.attr.command, command_file) command_arg = "source %s" % _runfiles(ctx, command_file) - files += [command_file] + files.append(command_file) ctx.actions.expand_template( template = ctx.file._template, @@ -492,7 +509,7 @@ def _k8s_test_namespace_impl(ctx): output = namespace_create, is_executable = True, ) - files += [namespace_create] + files.append(namespace_create) return [DefaultInfo( executable = namespace_create, @@ -533,16 +550,16 @@ def _k8s_test_setup_impl(ctx): for obj in ctx.attr.objects: if obj.files_to_run.executable: # add object' targets and excutables to runfiles - files += [obj.files_to_run.executable] + files.append(obj.files_to_run.executable) transitive.append(obj.default_runfiles.files) # add object' execution command - commands += [_runfiles(ctx, obj.files_to_run.executable) + " | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -"] + commands.append(_runfiles(ctx, obj.files_to_run.executable) + " | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -") else: files += obj.files.to_list() commands += [ctx.executable._template_engine.short_path + " --template=" + filename.short_path + " --variable=NAMESPACE=${NAMESPACE} | ${SET_NAMESPACE} $NAMESPACE | ${IT_MANIFEST_FILTER} | ${KUBECTL} apply -f -" for filename in obj.files.to_list()] - files += [ctx.executable._template_engine] + files.append(ctx.executable._template_engine) # create namespace script ctx.actions.expand_template(