From 7969040a5d836ea8bdea8955b44ea5ec3d08aec7 Mon Sep 17 00:00:00 2001 From: Sascha Schwarze Date: Fri, 27 Dec 2024 16:47:07 +0100 Subject: [PATCH] Test connectivity to the Git or Registry host before accessing the source This aims to mitigate rare failures where a BuildRun pod runs in a namespace with NetworkPolicies present. Kubernetes defines that isolation rules are guaranteed to be applied before any container of the Pod is started, but allow rules are eventually applied. Signed-off-by: Sascha Schwarze --- cmd/bundle/main.go | 7 +++++ cmd/git/main.go | 7 +++++ pkg/git/git.go | 26 ++++++++++++++++ pkg/git/git_test.go | 21 +++++++++++++ pkg/image/endpoint.go | 48 +++++++++++++++++++++++++++++ pkg/image/endpoint_test.go | 32 +++++++++++++++++++ pkg/util/tcp.go | 30 ++++++++++++++++++ pkg/util/tcp_test.go | 61 +++++++++++++++++++++++++++++++++++++ pkg/util/util_suite_test.go | 17 +++++++++++ 9 files changed, 249 insertions(+) create mode 100644 pkg/image/endpoint.go create mode 100644 pkg/image/endpoint_test.go create mode 100644 pkg/util/tcp.go create mode 100644 pkg/util/tcp_test.go create mode 100644 pkg/util/util_suite_test.go diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index a6cc065905..a4a4d0c153 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -73,6 +73,13 @@ func Do(ctx context.Context) error { return fmt.Errorf("mandatory flag --image is not set") } + // sanity-check the endpoint, if hostname extraction fails, ignore that failure + if hostname, port, err := image.ExtractHostnamePort(flagValues.image); err == nil { + if !util.TestConnection(hostname, port, 9) { + log.Printf("Warning: a connection test to %s:%d failed. The operation will likely fail.\n", hostname, port) + } + } + ref, err := name.ParseReference(flagValues.image) if err != nil { return err diff --git a/cmd/git/main.go b/cmd/git/main.go index ace87b77a7..7efbf08b55 100644 --- a/cmd/git/main.go +++ b/cmd/git/main.go @@ -164,6 +164,13 @@ func runGitClone(ctx context.Context) error { return &ExitError{Code: 101, Message: "the 'target' argument must not be empty"} } + // sanity-check the endpoint, if hostname extraction fails, ignore that failure + if hostname, port, err := shpgit.ExtractHostnamePort(flagValues.url); err == nil { + if !util.TestConnection(hostname, port, 9) { + log.Printf("Warning: a connection test to %s:%d failed. The operation will likely fail.\n", hostname, port) + } + } + if err := clone(ctx); err != nil { return err } diff --git a/pkg/git/git.go b/pkg/git/git.go index 28942ebd2e..66ed007726 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -23,6 +23,32 @@ const ( gitProtocol = "ssh" ) +// ExtractHostnamePort extracts the hostname and port of the provided Git URL +func ExtractHostnamePort(url string) (string, int, error) { + endpoint, err := transport.NewEndpoint(url) + if err != nil { + return "", 0, err + } + + port := endpoint.Port + + if port == 0 { + switch endpoint.Protocol { + case httpProtocol: + port = 80 + case httpsProtocol: + port = 443 + case gitProtocol: + port = 22 + + default: + return "", 0, fmt.Errorf("Unknown protocol: %s", endpoint.Protocol) + } + } + + return endpoint.Host, port, nil +} + // ValidateGitURLExists validate if a source URL exists or not // Note: We have an upcoming PR for the Build Status, where we // intend to define a single Status.Reason in the form of 'remoteRepositoryUnreachable', diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go index 9ffebc5dee..facd7df423 100644 --- a/pkg/git/git_test.go +++ b/pkg/git/git_test.go @@ -16,6 +16,27 @@ import ( var _ = Describe("Git", func() { + DescribeTable("the extraction of hostname and port", + func(url string, expectedHost string, expectedPort int, expectError bool) { + host, port, err := git.ExtractHostnamePort(url) + if expectError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).ToNot(HaveOccurred()) + Expect(host).To(Equal(expectedHost), "for "+url) + Expect(port).To(Equal(expectedPort), "for "+url) + } + }, + Entry("Check heritage SSH URL with default port", "ssh://github.com/shipwright-io/build.git", "github.com", 22, false), + Entry("Check heritage SSH URL with custom port", "ssh://github.com:12134/shipwright-io/build.git", "github.com", 12134, false), + Entry("Check SSH URL with default port", "git@github.com:shipwright-io/build.git", "github.com", 22, false), + Entry("Check HTTP URL with default port", "http://github.com/shipwright-io/build.git", "github.com", 80, false), + Entry("Check HTTPS URL with default port", "https://github.com/shipwright-io/build.git", "github.com", 443, false), + Entry("Check HTTPS URL with custom port", "https://github.com:9443/shipwright-io/build.git", "github.com", 9443, false), + Entry("Check HTTPS URL with credentials", "https://somebody:password@github.com/shipwright-io/build.git", "github.com", 443, false), + Entry("Check invalid URL", "ftp://github.com/shipwright-io/build", "", 0, true), + ) + DescribeTable("the source url validation errors", func(url string, expected types.GomegaMatcher) { Expect(git.ValidateGitURLExists(context.TODO(), url)).To(expected) diff --git a/pkg/image/endpoint.go b/pkg/image/endpoint.go new file mode 100644 index 0000000000..b5a63b5b9e --- /dev/null +++ b/pkg/image/endpoint.go @@ -0,0 +1,48 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "fmt" + "strconv" + "strings" + + "github.com/google/go-containerregistry/pkg/name" +) + +// ExtractHostnamePort tries to extract the hostname and port of the provided image URL +func ExtractHostnamePort(url string) (string, int, error) { + ref, err := name.ParseReference(url) + if err != nil { + return "", 0, err + } + + registry := ref.Context().Registry + host := registry.RegistryStr() + hostname := host + port := 0 + + parts := strings.SplitN(host, ":", 2) + if len(parts) == 2 { + hostname = parts[0] + if port, err = strconv.Atoi(parts[1]); err != nil { + return "", 0, err + } + } else { + scheme := registry.Scheme() + + switch scheme { + case "http": + port = 80 + case "https": + port = 443 + + default: + return "", 0, fmt.Errorf("Unknown protocol: %s", scheme) + } + } + + return hostname, port, nil +} diff --git a/pkg/image/endpoint_test.go b/pkg/image/endpoint_test.go new file mode 100644 index 0000000000..52ac72e406 --- /dev/null +++ b/pkg/image/endpoint_test.go @@ -0,0 +1,32 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package image_test + +import ( + "github.com/shipwright-io/build/pkg/image" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Endpoints", func() { + + DescribeTable("the extraction of hostname and port", + func(url string, expectedHost string, expectedPort int, expectError bool) { + host, port, err := image.ExtractHostnamePort(url) + if expectError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).ToNot(HaveOccurred()) + Expect(host).To(Equal(expectedHost), "for "+url) + Expect(port).To(Equal(expectedPort), "for "+url) + } + }, + Entry("Check a URL with default port", "registry.access.redhat.com/ubi9/ubi-minimal", "registry.access.redhat.com", 443, false), + Entry("Check a URL with custom port", "registry.access.redhat.com:9443/ubi9/ubi-minimal", "registry.access.redhat.com", 9443, false), + Entry("Check a URL without host", "ubuntu", "index.docker.io", 443, false), + Entry("Check invalid URL", "ftp://registry.access.redhat.com/ubi9/ubi-minimal", "", 0, true), + ) +}) diff --git a/pkg/util/tcp.go b/pkg/util/tcp.go new file mode 100644 index 0000000000..d3dda280f1 --- /dev/null +++ b/pkg/util/tcp.go @@ -0,0 +1,30 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "fmt" + "net" + "time" +) + +// TestConnection tries to establish a connection to a provided host using a 5 seconds timeout. +func TestConnection(hostname string, port int, retries int) bool { + host := fmt.Sprintf("%s:%d", hostname, port) + + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + } + + for i := 0; i <= retries; i++ { + conn, _ := dialer.Dial("tcp", host) + if conn != nil { + _ = conn.Close() + return true + } + } + + return false +} diff --git a/pkg/util/tcp_test.go b/pkg/util/tcp_test.go new file mode 100644 index 0000000000..fca59c265a --- /dev/null +++ b/pkg/util/tcp_test.go @@ -0,0 +1,61 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package util_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/shipwright-io/build/pkg/util" +) + +var _ = Describe("TCP", func() { + + Context("TestConnection", func() { + + var result bool + var hostname string + var port int + + JustBeforeEach(func() { + result = util.TestConnection(hostname, port, 1) + }) + + Context("For a broken endpoint", func() { + + BeforeEach(func() { + hostname = "shipwright.io" + port = 33333 + }) + + It("returns false", func() { + Expect(result).To(BeFalse()) + }) + }) + + Context("For an unknown host", func() { + + BeforeEach(func() { + hostname = "shipwright-dhasldglidgewidgwd.io" + port = 33333 + }) + + It("returns false", func() { + Expect(result).To(BeFalse()) + }) + }) + + Context("For a functional endpoint", func() { + + BeforeEach(func() { + hostname = "github.com" + port = 443 + }) + + It("returns true", func() { + Expect(result).To(BeTrue()) + }) + }) + }) +}) diff --git a/pkg/util/util_suite_test.go b/pkg/util/util_suite_test.go new file mode 100644 index 0000000000..3c3318b8cc --- /dev/null +++ b/pkg/util/util_suite_test.go @@ -0,0 +1,17 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package util_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestGit(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Util Suite") +}