Refactor Shipyard to be consumable by other projects

.dapper

@@ -46,7 +46,7 @@ do
             set -x
             ;;
         --version|-v)
-            echo Submariner Dapper
+            echo Shipyard Dapper
             exit 0
             ;;
         --mount-suffix|-S)
@@ -69,13 +69,13 @@ done
 
 [ -n "$command" ] && set -- "$command"
 
+buildargs=(--build-arg "ORG=${ORG}" --build-arg "PROJECT=${PROJECT}")
+[ -n "${SHIPYARD_REPO}" ] && buildargs+=(--build-arg "SHIPYARD_REPO=${SHIPYARD_REPO}")
+[ -n "${SHIPYARD_TAG}" ] && buildargs+=(--build-arg "SHIPYARD_TAG=${SHIPYARD_TAG}")
 gitid="$(git symbolic-ref --short HEAD 2>/dev/null | tr / _ || :)"
 gitid="${gitid:-$(git show --format=%h -s)}"
 container="$(basename "$(pwd)"):${gitid}"
-DOCKER_BUILDKIT=1 docker build -t "${container}" -f "${file}" \
-       --build-arg "BASE_BRANCH=${BASE_BRANCH:-devel}" \
-       --build-arg "PROJECT=${PROJECT}" \
-       .
+DOCKER_BUILDKIT=1 docker build -t "${container}" -f "${file}" "${buildargs[@]}" .
 
 extract_var() {
     docker inspect "$1" | grep "$2" | sed -E "s/.*\"$2=(.*)\",?/\1/;q"

.github/workflows/consuming.yml

@@ -49,7 +49,7 @@ jobs:
         run: make -C shipyard images
 
       - name: Copy Shipyard resources
-        run: cp -n shipyard/{Dockerfile.*,Makefile.dapper,.dapper} .
+        run: cp -n shipyard/{Dockerfile.*,Makefile.dapper,Makefile.shipyard,.dapper} .
 
       - name: Make sure ${{ matrix.project }} is using the built Shipyard image
         run: sed -i 's/shipyard-dapper-base:*.*/shipyard-dapper-base:dev/' Dockerfile.dapper
@@ -97,7 +97,7 @@ jobs:
           path: ${{ matrix.project }}
 
       - name: Copy Shipyard resources
-        run: cp -n Dockerfile.* Makefile.dapper .dapper ${{ matrix.project }}/
+        run: cp -n Dockerfile.* Makefile.dapper Makefile.shipyard .dapper ${{ matrix.project }}/
 
       - name: Make sure ${{ matrix.project }} is using the built Shipyard image
         run: sed -i 's/shipyard-dapper-base:*.*/shipyard-dapper-base:dev/' ${{ matrix.project }}/Dockerfile.dapper
@@ -152,7 +152,7 @@ jobs:
           path: ${{ matrix.project }}
 
       - name: Copy Shipyard resources
-        run: cp -n Dockerfile.* Makefile.dapper .dapper ${{ matrix.project }}/
+        run: cp -n Dockerfile.* Makefile.dapper* .dapper ${{ matrix.project }}/
 
       - name: Make sure ${{ matrix.project }} is using the built Shipyard image
         run: sed -i 's/shipyard-dapper-base:*.*/shipyard-dapper-base:dev/' ${{ matrix.project }}/Dockerfile.dapper

Dockerfile.dapper

@@ -1,9 +1,11 @@
-ARG BASE_BRANCH
-FROM quay.io/submariner/shipyard-dapper-base:${BASE_BRANCH}
+ARG SHIPYARD_REPO=quay.io/submariner
+ARG SHIPYARD_TAG=devel
+FROM ${SHIPYARD_REPO}/shipyard-dapper-base:${SHIPYARD_TAG}
 
+ARG ORG
 ARG PROJECT
 ENV DAPPER_ENV="CI QUAY_USERNAME QUAY_PASSWORD MAKEFLAGS PLUGIN TEST_ARGS E2E_TESTDIR GITHUB_USER GITHUB_TOKEN USING" \
-    DAPPER_SOURCE=/go/src/github.com/submariner-io/${PROJECT} DAPPER_DOCKER_SOCKET=true
+    DAPPER_SOURCE=/go/src/github.com/${ORG}/${PROJECT} DAPPER_DOCKER_SOCKET=true
 ENV OVN_DIR=${DAPPER_SOURCE}/ovn-kubernetes
 ENV DAPPER_OUTPUT=${DAPPER_SOURCE}/output
 

Dockerfile.linting

@@ -1,5 +1,6 @@
-ARG BASE_BRANCH
-FROM quay.io/submariner/shipyard-linting:${BASE_BRANCH}
+ARG SHIPYARD_REPO=quay.io/submariner
+ARG SHIPYARD_TAG=devel
+FROM ${SHIPYARD_REPO}/shipyard-linting:${SHIPYARD_TAG}
 
 ENV DAPPER_ENV="CI GITHUB_SHA MAKEFLAGS" \
     DAPPER_SOURCE=/opt/linting

Makefile

@@ -54,7 +54,7 @@ include Makefile.versions
 # Shipyard-specific starts
 # We need to ensure images, including the Shipyard base image, are updated
 # before we start Dapper
-clean-clusters cleanup cloud-prepare clusters deploy deploy-latest e2e golangci-lint post-mortem print-version scale unit upgrade-e2e: package/.image.shipyard-dapper-base
+clean-clusters cleanup cloud-prepare clusters deploy deploy-latest e2e golangci-lint post-mortem packagedoc-lint print-version scale unit upgrade-e2e: package/.image.shipyard-dapper-base
 deploy deploy-latest e2e upgrade-e2e: package/.image.nettest
 
 .DEFAULT_GOAL := lint

Makefile.clusters

@@ -0,0 +1,60 @@
+### VARIABLES ###
+
+, := ,
+USING = $(subst $(,), ,$(using))
+_using = ${USING}
+
+# General make flags/variables
+PARALLEL ?= true
+TIMEOUT ?= 5m
+export PARALLEL SETTINGS TIMEOUT
+
+# Flags affecting `make clusters`
+K8S_VERSION ?= 1.28
+METALLB_VERSION ?= 0.13.5
+OLM_VERSION ?= v0.18.3
+PROVIDER ?= kind
+export AIR_GAPPED DUAL_STACK K8S_VERSION LOAD_BALANCER METALLB_VERSION OLM OLM_VERSION OVERLAPPING PROMETHEUS PROVIDER
+
+### PROCESSING `using=` ###
+
+ifneq (,$(filter ocp,$(_using)))
+PROVIDER = ocp
+endif
+
+ifneq (,$(filter acm,$(_using)))
+PROVIDER = acm
+endif
+
+ifneq (,$(filter load-balancer,$(_using)))
+LOAD_BALANCER = true
+endif
+
+ifneq (,$(filter air-gap,$(_using)))
+AIR_GAPPED = true
+endif
+
+ifneq (,$(filter dual-stack,$(_using)))
+DUAL_STACK = true
+endif
+
+ifneq (,$(filter overlapping,$(_using)))
+OVERLAPPING = true
+endif
+
+ifneq (,$(filter prometheus,$(_using)))
+PROMETHEUS = true
+endif
+
+### TARGETS ###
+
+# [clean-clusters] removes running clusters
+clean-clusters:
+	$(SCRIPTS_DIR)/cleanup.sh
+cleanup: clean-clusters
+
+# [clusters] creates KIND clusters that can then be used to deploy Submariner
+clusters:
+	$(SCRIPTS_DIR)/$@.sh
+
+.PHONY: clean-clusters cleanup clusters

Makefile.dapper

@@ -1,72 +1,17 @@
 # This Makefile contains the rules required to set up our
 # Dapper-based build environment
 
-, := ,
+ORG ?= submariner-io
 PROJECT ?= $(notdir $(CURDIR))
-BASE_DAPPER := Dockerfile.dapper
-USING = $(subst $(,), ,$(using))
-LINTING_DAPPER := Dockerfile.linting
-LINTING_GOALS := gitlint shellcheck yamllint markdownlint
-NON_DAPPER_GOALS += .dapper shell targets $(LINTING_GOALS)
-export MAKEFLAGS
-export PROJECT
-export USING
+SHIPYARD_TAG ?= $(BASE_BRANCH)
+SHIPYARD_URL ?= https://raw.githubusercontent.com/submariner-io/shipyard/$(BASE_BRANCH)
+export ORG PROJECT SHIPYARD_TAG SHIPYARD_URL
 
-# Define LOCAL_BUILD to build directly on the host and not inside a Dapper container
-ifdef LOCAL_BUILD
-DAPPER_HOST_ARCH ?= $(shell go env GOHOSTARCH)
-SHIPYARD_DIR ?= ../shipyard
-SCRIPTS_DIR ?= $(SHIPYARD_DIR)/scripts/shared
-
-export DAPPER_HOST_ARCH
-export SHIPYARD_DIR
-export SCRIPTS_DIR
-
-RUN_IN_DAPPER :=
-
-.dapper:
-
-else
-
-.dapper:
-	@echo Downloading dapper
-	@curl -sfLO https://raw.githubusercontent.com/submariner-io/shipyard/$(BASE_BRANCH)/$@
-	@chmod +x .dapper
-	@./.dapper -v
-
-SELINUX_CONTEXT := $(shell (selinuxenabled && echo -S z) 2>/dev/null)
-RUN_IN_DAPPER = ./.dapper $(DAPPER_ARGS) $(SELINUX_CONTEXT) --
-
-endif
-
-ifeq (true,$(DEBUG_PRINT))
-MAKE_DEBUG_FLAG = --debug=b
-endif
-
-USING = $(subst $(,), ,$(using))
-_using = ${USING}
-
-ifneq (,$(filter dual-stack,$(_using)))
-IPV6_FLAGS = --ipv6 --subnet fc00:1234:4444::/64
-endif
-
-# Only run command line goals in dapper (except things that have to run outside of dapper).
-# Otherwise, make applies this rule to various files and tries to build them in dapper (which doesn't work, obviously).
-$(filter-out .dapper prune-images shell targets $(NON_DAPPER_GOALS),$(MAKECMDGOALS)): .dapper $(BASE_DAPPER)
-	@[ -z "$$CI" ] || echo "::group::Launching a container to run 'make $@'"
-	-docker network create $(IPV6_FLAGS) -d bridge kind
-	+$(RUN_IN_DAPPER) make $(MAKE_DEBUG_FLAG) $@
-
-# The original dockerfiles will live in Shipyard and be downloaded by consuming projects.
-$(BASE_DAPPER) $(LINTING_DAPPER):
+Makefile.shipyard:
 	@echo Downloading $@
-	@curl -sfLO https://raw.githubusercontent.com/submariner-io/shipyard/$(BASE_BRANCH)/$@
+	@curl -sfLO $(SHIPYARD_URL)/$@
 
-# Run silently as the commands are pretty straightforward and `make` hasn't a lot to do
-$(LINTING_GOALS): DAPPER_ARGS := -f $(LINTING_DAPPER)
-$(LINTING_GOALS): .dapper $(LINTING_DAPPER)
-	@[ -z "$$CI" ] || echo "::group::Launching a container to run 'make $@'"
-	@$(RUN_IN_DAPPER) make $@
+include Makefile.shipyard
 
 # [prune-images] removes all Submariner-provided images and all untagged images
 # Use this to ensure you use current images
@@ -81,14 +26,4 @@ prune-images:
 	    fi \
 	done
 
-shell: DAPPER_ARGS := -s
-shell: .dapper $(BASE_DAPPER)
-	$(RUN_IN_DAPPER)
-
-# Run silently to just list the targets (hence we can't use the generic dapper wrapper recipe).
-# This only lists targets accessible inside dapper (which are 99% of targets we use)
-targets: DAPPER_ARGS := -f $(LINTING_DAPPER)
-targets: $(LINTING_DAPPER)
-	@$(RUN_IN_DAPPER) eval "\$${SCRIPTS_DIR}/targets.sh"
-
-.PHONY: prune-images shell targets $(LINTING_GOALS)
+.PHONY: prune-images