Skip to content

Commit

Permalink
build: create Docker images automatically
Browse files Browse the repository at this point in the history
  • Loading branch information
kris7t committed Jun 24, 2024
1 parent d5efb58 commit d46a69a
Show file tree
Hide file tree
Showing 11 changed files with 241 additions and 121 deletions.
103 changes: 85 additions & 18 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ on:
jobs:
build:
name: Build
permissions:
contents: read
strategy:
matrix:
os:
Expand All @@ -32,10 +34,6 @@ jobs:
if [ "${SONAR_TOKEN}" != '' ]; then
echo 'is_SONAR_TOKEN_set=true' >> $GITHUB_OUTPUT
fi
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: ${{ !steps.check-secret.outputs.is_SONAR_TOKEN_set && 1 || 0 }} # Shallow clones should be disabled for a better relevancy of SonarCloud analysis
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
Expand All @@ -56,6 +54,10 @@ jobs:
~/.sonar/cache
key: ${{ matrix.os }}-sonar
restore-keys: ${{ matrix.os }}-sonar
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: ${{ !steps.check-secret.outputs.is_SONAR_TOKEN_set && 1 || 0 }} # Shallow clones should be disabled for a better relevancy of SonarCloud analysis
- name: Cache node distribution
uses: actions/cache@v4
with:
Expand All @@ -81,15 +83,15 @@ jobs:
run: |
./gradlew sonar -Pci --info --stacktrace --max-workers 4 --no-daemon
- name: Build signed Maven repository
if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' && github.repository == 'graphs4value/refinery' }}
if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' && github.repository_owner == 'graphs4value' }}
env:
PGP_KEY: ${{ secrets.PGP_KEY }}
PGP_KEY_ID: ${{ secrets.PGP_KEY_ID }}
PGP_PASSWORD: ${{ secrets.PGP_PASSWORD }}
run: |
./gradlew mavenRepositoryTar -Pci -PforceSign --info --stacktrace --max-workers 4 --no-daemon
- name: Build unsigned Maven repository
if: ${{ matrix.os == 'ubuntu-latest' && (github.event_name != 'push' || github.repository != 'graphs4value/refinery') }}
if: ${{ matrix.os == 'ubuntu-latest' && (github.event_name != 'push' || github.repository_owner != 'graphs4value') }}
run: |
./gradlew mavenRepositoryTar -Pci --info --stacktrace --max-workers 4 --no-daemon
- name: Upload Maven repository artifact
Expand All @@ -99,6 +101,14 @@ jobs:
name: maven-repository-tar
path: build/refinery-maven-repository.tar
compression-level: 0
- name: Upload application artifacts
if: ${{ matrix.os == 'ubuntu-latest' }}
uses: actions/upload-artifact@v4
with:
name: distributions-tar
path: subprojects/**/build/distributions/*.tar
compression-level: 0
retention-days: 5 # No need to preserve for long, since they are uploaded to GHCR
- name: Upload site artifact
if: ${{ matrix.os == 'ubuntu-latest' }}
uses: actions/upload-artifact@v4
Expand All @@ -108,9 +118,12 @@ jobs:
compression-level: 0
reuse-check:
name: REUSE Compliance Check
permissions:
contents: read
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Checkout code
uses: actions/checkout@v4
- name: REUSE Compliance Check
uses: fsfe/reuse-action@a46482ca367aef4454a87620aa37c2be4b2f8106
with:
Expand Down Expand Up @@ -140,25 +153,79 @@ jobs:
git_config_global: true
git_user_signingkey: true
git_commit_gpgsign: true
- name: Commit and push to graphs4value.github.io
env:
GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }}
GITHUB_REPOSITORY: ${{ github.sha }}
GITHUB_SHA: ${{ github.sha }}
- name: Create empty git repository
run: |
mkdir graphs4value.github.io
pushd graphs4value.github.io
cd graphs4value.github.io
git config --global init.defaultBranch main
git config --global user.name "Graphs4Value bot"
git config --global user.email "refinery@refinery.tools"
git init
git remote add origin "https://x-access-token:${GH_PAGES_TOKEN}@github.com/graphs4value/graphs4value.github.io.git"
- name: Extract site artifact
working-directory: ./graphs4value.github.io
run: |
unzip ../site-zip/refinery-docs.zip
- name: Extract Maven repository artifact
working-directory: ./graphs4value.github.io
run: |
mkdir -p maven/snapshots
pushd maven/snapshots
tar xf ../../../maven-repository-tar/refinery-maven-repository.tar
popd
cd maven/snapshots
tar -xvf ../../../maven-repository-tar/refinery-maven-repository.tar
- name: Commit and push to graphs4value.github.io
working-directory: ./graphs4value.github.io
env:
GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_SHA: ${{ github.sha }}
run: |
git remote add origin "https://x-access-token:${GH_PAGES_TOKEN}@github.com/graphs4value/graphs4value.github.io.git"
git add .
git commit -S -m "Update from https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}"
git push --force origin main
popd
docker-build:
name: Build Docker images
needs: build
permissions:
packages: write
contents: read
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
with:
platforms: arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb
with:
platforms: linux/amd64,linux/arm64
- name: Checkout code
uses: actions/checkout@v4
- name: Download application artifacts
uses: actions/download-artifact@v4
with:
name: distributions-tar
path: subprojects
- name: Extract application artifacts
working-directory: ./docker
run: |
./prepare_context.sh
- name: Bake images
working-directory: ./docker
run: |
./bake.sh false --set '*.cache-from=gha' --set '*.cache-to=type=gha,mode=max'
- name: Log in to GitHub Container registry
if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'graphs4value/refinery' }}
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Upload images to GitHub Container registry
if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'graphs4value/refinery' }}
working-directory: ./docker
run: |
./bake.sh true --set '*.cache-from=gha' --set '*.cache-to=type=gha,mode=max'
- name: Delete application artifacts
uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1
with:
name: distributions-tar
5 changes: 5 additions & 0 deletions docker/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# SPDX-FileCopyrightText: 2024 The Refinery Authors
#
# SPDX-License-Identifier: CC0-1.0

context/extracted
17 changes: 17 additions & 0 deletions docker/bake.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env bash

# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
#
# SPDX-License-Identifier: EPL-2.0

set -euo pipefail

REFINERY_VERSION="$(./get_version.sh)"
export REFINERY_VERSION

export REFINERY_PUSH="${1-false}"

export SOURCE_DATE_EPOCH=0

cd context
exec docker buildx bake -f docker-bake.hcl "${@:2}"
48 changes: 3 additions & 45 deletions docker/build.sh
Original file line number Diff line number Diff line change
@@ -1,55 +1,13 @@
#!/usr/bin/env bash

# SPDX-FileCopyrightText: 2023-2024 The Refinery Authors <https://refinery.tools/>
# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
#
# SPDX-License-Identifier: EPL-2.0

set -euo pipefail

(cd .. && ./gradlew distTar)

refinery_version="$(grep '^version=' ../gradle.properties | cut -d'=' -f2)"
cli_distribution_name="refinery-generator-cli-${refinery_version}"
web_distribution_name="refinery-language-web-${refinery_version}"
./prepare_context.sh

rm -rf "${cli_distribution_name}" "${web_distribution_name}" {cli,web}_dist \
{cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin

tar -xf "../subprojects/generator-cli/build/distributions/${cli_distribution_name}.tar"
mv "${cli_distribution_name}" cli_dist
tar -xf "../subprojects/language-web/build/distributions/${web_distribution_name}.tar"
mv "${web_distribution_name}" web_dist
mkdir -p {cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin

# Our application itself is very small, so it will get added as the last layer
# of both containers.
mv cli_dist/lib/refinery-* cli_app_lib
mv web_dist/lib/refinery-* web_app_lib

for i in cli_dist/lib/*; do
j="web${i#cli}"
if [[ -f "$j" ]]; then
mv "$i" "common_lib${i#cli_dist/lib}"
rm "$j"
fi
done

# Move architecture-specific jars to their repsective directories.
mv common_lib/ortools-linux-x86-64-*.jar common_amd64_lib
mv common_lib/ortools-linux-aarch64-*.jar common_arm64_lib
rm common_lib/ortools-{darwin,win32}-*.jar
# Move the applications jars for the dependencies into a separate Docker layer
# to enable faster updates.
mv cli_dist/lib/* cli_lib
mv web_dist/lib/* web_lib
# Omit references to jars not present for the current architecture from the
# startup scripts.
sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_amd64_bin/refinery-generator-cli
sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_arm64_bin/refinery-generator-cli
chmod a+x cli_{amd64,arm64}_bin/refinery-generator-cli
sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_amd64_bin/refinery-language-web
sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_arm64_bin/refinery-language-web
chmod a+x web_{amd64,arm64}_bin/refinery-language-web
rm -rf {cli,web}_dist

REFINERY_VERSION="${refinery_version}" docker buildx bake -f docker-bake.hcl
./bake.sh "${1-false}" "${@:2}"
11 changes: 5 additions & 6 deletions docker/Dockerfile.base → docker/context/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ RUN apt-get update && \
ln -sf /usr/bin/x86_64-linux-gnu-objcopy /usr/bin/objcopy && \
rm -rf /var/lib/apt/lists/*

FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH as jlink
RUN jlink --no-header-files --no-man-pages --compress=2 \
FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH AS jlink
RUN jlink --no-header-files --no-man-pages \
--module-path=/crossjdk/jmods --strip-debug --add-modules \
java.base,java.logging,java.xml,jdk.zipfs \
--output /jlink
Expand All @@ -44,15 +44,14 @@ RUN dnf install -y findutils && \
COPY --link --from=jlink /jlink /usr/lib/java
ENV JAVA_HOME="/usr/lib/java" PATH="/usr/lib/java/bin:${PATH}"
# Layer with platform-independent dependencies, slow changing.
ADD --link common_lib /app/lib
ADD --link extracted/common_lib /app/lib

FROM base AS base-amd64
# Layer with platform-dependent dependencies, slow changing.
ADD --link common_amd64_lib /app/lib
ADD --link extracted/common_amd64_lib /app/lib

FROM base AS base-arm64
# Layer with platform-dependent dependencies, slow changing.
ADD --link common_arm64_lib /app/lib
ADD --link extracted/common_arm64_lib /app/lib

FROM base-$TARGETARCH

8 changes: 4 additions & 4 deletions docker/Dockerfile.cli → docker/context/Dockerfile.cli
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,21 @@

FROM base AS cli-base
# Layer with platform-dependent dependencies, slow changing.
ADD --link cli_lib /app/lib
ADD --link extracted/cli_lib /app/lib

FROM cli-base AS cli-amd64
# Layer with platform-dependent startup script containing references to all
# dependency versions.
ADD --link cli_amd64_bin /app/bin
ADD --link extracted/cli_amd64_bin /app/bin

FROM cli-base AS cli-arm64
# Layer with platform-dependent startup script containing references to all
# dependency versions.
ADD --link cli_arm64_bin /app/bin
ADD --link extracted/cli_arm64_bin /app/bin

FROM cli-$TARGETARCH
# Layer with platform-independent application jars.
ADD --link cli_app_lib /app/lib
ADD --link extracted/cli_app_lib /app/lib
# Common settings added on top.
ENV REFINERY_LIBRARY_PATH=/data
USER 1000
Expand Down
8 changes: 4 additions & 4 deletions docker/Dockerfile.web → docker/context/Dockerfile.web
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,21 @@

FROM base AS web-base
# Layer with platform-dependent dependencies, slow changing.
ADD --link web_lib /app/lib
ADD --link extracted/web_lib /app/lib

FROM web-base AS web-amd64
# Layer with platform-dependent startup script containing references to all
# dependency versions.
ADD --link web_amd64_bin /app/bin
ADD --link extracted/web_amd64_bin /app/bin

FROM web-base AS web-arm64
# Layer with platform-dependent startup script containing references to all
# dependency versions.
ADD --link web_arm64_bin /app/bin
ADD --link extracted/web_arm64_bin /app/bin

FROM web-$TARGETARCH
# Layer with platform-independent application jars.
ADD --link web_app_lib /app/lib
ADD --link extracted/web_app_lib /app/lib
# Common settings added on top.
ENV REFINERY_LISTEN_HOST=0.0.0.0 REFINERY_LISTEN_PORT=8888
EXPOSE 8888
Expand Down
43 changes: 43 additions & 0 deletions docker/context/docker-bake.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
#
# SPDX-License-Identifier: EPL-2.0

variable "REFINERY_VERSION" {
default = ""
}

variable "REFINERY_PUSH" {
default = "false"
}

group "default" {
targets = ["cli", "web"]
}

target "base" {
dockerfile = "Dockerfile.base"
platforms = ["linux/amd64", "linux/arm64"]
output = ["type=cacheonly"]
}

target "cli" {
dockerfile = "Dockerfile.cli"
platforms = ["linux/amd64", "linux/arm64"]
output = [
"type=image,push=${REFINERY_PUSH},\"name=ghcr.io/graphs4value/refinery-cli:${REFINERY_VERSION},ghcr.io/graphs4value/refinery-cli:latest\",annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,\"annotation-index.org.opencontainers.image.description=Command line interface for Refinery, an efficient graph solver for generating well-formed models\",annotation-index.org.opencontainers.image.licenses=EPL-2.0"
]
contexts = {
base = "target:base"
}
}

target "web" {
dockerfile = "Dockerfile.web"
platforms = ["linux/amd64", "linux/arm64"]
output = [
"type=image,push=${REFINERY_PUSH},\"name=ghcr.io/graphs4value/refinery:${REFINERY_VERSION},ghcr.io/graphs4value/refinery:latest\",annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,annotation-index.org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models,annotation-index.org.opencontainers.image.licenses=EPL-2.0"
]
contexts = {
base = "target:base"
}
}
Loading

0 comments on commit d46a69a

Please sign in to comment.