Merge pull request #13 from esgf-nimbus/update-containers

Update containers

.github/workflows/containers.yaml

@@ -18,116 +18,197 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  detect-changes:
+  get-issue-number:
     runs-on: ubuntu-latest
-    name: Detect changes to Dockerfiles
+    name: Get PR number
     permissions:
-      pull-requests: read
+      contents: read
     outputs:
-      modified_files_count: ${{ steps.changed-files.outputs.modified_files_count }}
-      other_modified_files_count: ${{ steps.changed-files.outputs.other_modified_files_count }}
-      directories: ${{ steps.format.outputs.directories }}
+      pr-number: "${{ steps.pr-number.outputs.pr-number }}"
     steps:
-      - uses: actions/checkout@v3
-      - name: Detect changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v37
+      - uses: actions/github-script@v6
+        id: get-issue-number
         with:
-          path: dockerfiles
-          dir_names: true
-          dir_names_max_depth: 1
-          dir_names_exclude_current_dir: true
-          files: minimal-notebook/**
-      - id: format
-        run: |
-          echo "${{ steps.changed-files.outputs.other_modified_files }}" | python -c "import sys,json;d=json.dumps({'directories': sys.stdin.read().strip().split(' ')});print(f'directories={d}')" >> $GITHUB_OUTPUT
+          script: |
+            if (context.issue.number) {
+              return context.issue.number;
+            } else {
+              return (
+                await github.rest.repos.listPullRequestAssociatedWithCommit({
+                  commit_sha: context.sha,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                })
+              ).data[0].number;
+            }
+          result-encoding: string
+      - id: pr-number
+        run: echo "pr-number=${{ steps.get-issue-number.outputs.result }}" >> $GITHUB_OUTPUT
 
-  build-base-container:
-    if: needs.detect-changes.outputs.modified_files_count > 0
+  build-minimal-notebook:
     needs:
-      - detect-changes
+      - get-issue-number
     runs-on: ubuntu-latest
-    name: Build base container
+    name: Build minimal-notebook
     permissions:
       packages: write
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
       - id: get-version
         run: |
-          pip install tbump
-          pushd dockerfiles/minimal-notebook
-          echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
-      - uses: docker/build-push-action@v4
-        if: github.event_name == 'pull_request'
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "version=pr-${{ needs.get-issue-number.outputs.pr-number }}" >> $GITHUB_OUTPUT
+          else
+            pip install tbump
+            pushd dockerfiles/minimal-notebook
+            echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
+          fi
+      - uses: docker/build-push-action@v5
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           context: dockerfiles/minimal-notebook
-          pull: true
           push: true
-          tags: ghcr.io/esgf-nimbus/minimal-notebook:cache
-      - uses: docker/build-push-action@v4
-        if: github.event_name == 'push'
+          tags: ghcr.io/esgf-nimbus/minimal-notebook:${{ steps.get-version.outputs.version }}
+
+  build-climate-notebook:
+    needs:
+      - get-issue-number
+      - build-minimal-notebook
+    runs-on: ubuntu-latest
+    name: Build climate-notebook
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - id: get-version
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            VERSION=pr-${{ needs.get-issue-number.outputs.pr-number }}
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+            echo "build-args=TAG=$VERSION" >> $GITHUB_OUTPUT
+          else
+            pip install tbump
+            pushd dockerfiles/climate-notebook
+            echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
+            echo "build-args=" >> $GITHUB_OUTPUT
+          fi
+      - uses: docker/build-push-action@v5
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          context: dockerfiles/minimal-notebook
-          pull: true
+          context: dockerfiles/climate-notebook
           push: true
-          tags: ghcr.io/esgf-nimbus/minimal-notebook:${{ steps.get-version.outputs.version }}
+          tags: ghcr.io/esgf-nimbus/climate-notebook:${{ steps.get-version.outputs.version }}
+          build-args: "${{ steps.get-version.outputs.build-args }}"
 
-  build-other-containers:
-    if: always() && needs.detect-changes.outputs.other_modified_files_count > 0
+  build-climate-notebook-gpu:
     needs:
-      - detect-changes
-      - build-base-container
+      - get-issue-number
+      - build-climate-notebook
     runs-on: ubuntu-latest
-    name: Build other containers
+    name: Build climate-notebook-gpu
     permissions:
       packages: write
-    strategy:
-      matrix: ${{ fromJSON(needs.detect-changes.outputs.directories) }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
       - id: get-version
         run: |
-          pip install tbump
-          pushd dockerfiles/${{ matrix.directories }}
-          echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
-      - uses: docker/build-push-action@v4
-        if: github.event_name == 'pull_request'
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            VERSION=pr-${{ needs.get-issue-number.outputs.pr-number }}
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+            echo "build-args=TAG=$VERSION" >> $GITHUB_OUTPUT
+          else
+            pip install tbump
+            pushd dockerfiles/climate-notebook-gpu
+            echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
+            echo "build-args=" >> $GITHUB_OUTPUT
+          fi
+      - uses: docker/build-push-action@v5
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          context: dockerfiles/${{ matrix.directories }}
-          build-args: ${{ (contains(matrix.directories, 'minimal-notebook') || contains(matrix.directories, 'nimbus')) && 'TAG=cache' || '' }}
-          pull: true
+          context: dockerfiles/climate-notebook-gpu
           push: true
-          tags: ghcr.io/esgf-nimbus/${{ matrix.directories }}:cache
-      - uses: docker/build-push-action@v4
-        if: github.event_name == 'push'
+          tags: ghcr.io/esgf-nimbus/climate-notebook-gpu:${{ steps.get-version.outputs.version }}
+          build-args: "${{ steps.get-version.outputs.build-args }}"
+
+  other-containers:
+    runs-on: ubuntu-latest
+    name: Get other containers
+    outputs:
+      directories: "${{ steps.get-paths.outputs.directories }}"
+    steps:
+      - uses: actions/checkout@v4
+      - id: get-paths
+        run: |
+          CONTAINERS="$(ls dockerfiles/ | grep -v -E "Makefile|climate*|minimal" | jq -R -s -c 'split("\n")[:-1]')"
+
+          echo "directories=$CONTAINERS" >> $GITHUB_OUTPUT
+
+  build-other-containers:
+    needs:
+      - get-issue-number
+      - other-containers
+    runs-on: ubuntu-latest
+    name: Build other containers
+    permissions:
+      packages: write
+    strategy:
+      matrix:
+        directory: ${{ fromJSON(needs.other-containers.outputs.directories) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - id: get-version
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            VERSION=pr-${{ needs.get-issue-number.outputs.pr-number }}
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+          else
+            pip install tbump
+            pushd dockerfiles/${{ matrix.directory }}
+            echo "version=$(tbump current-version)" >> $GITHUB_OUTPUT
+          fi
+      - uses: docker/build-push-action@v5
         with:
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          context: dockerfiles/${{ matrix.directories }}
-          pull: true
+          context: dockerfiles/${{ matrix.directory }}
           push: true
-          tags: ghcr.io/esgf-nimbus/${{ matrix.directories }}:${{ steps.get-version.outputs.version }}
+          tags: ghcr.io/esgf-nimbus/${{ matrix.directory }}:${{ steps.get-version.outputs.version }}

dockerfiles/Makefile

@@ -5,8 +5,7 @@ NAMESPACE ?= ghcr.io/esgf-nimbus
 build push run: REPOSITORY = $(lastword $(subst /, ,$(PWD)))
 
 build: changelog
-	docker build $(ARGS) -t $(NAMESPACE)/$(REPOSITORY):$(VERSION) . && \
-		$(POST_BUILD)
+	docker build $(ARGS) -t $(NAMESPACE)/$(REPOSITORY):$(VERSION) . $(if $(POST_BUILD),&& $(POST_BUILD),)
 
 push:
 	docker push $(NAMESPACE)/$(REPOSITORY):$(VERSION)
@@ -15,7 +14,7 @@ run:
 	docker run -it $(ARGS) -v $(PWD):/host $(if $(ENTRYPOINT),--entrypoint $(ENTRYPOINT),) $(NAMESPACE)/$(REPOSITORY):$(VERSION) $(COMMAND)
 
 changelog:
-	git log --pretty="%s [%h](https://github.com/esgf-nimbus/nimbus/commit/%h) (%cs)" . | python -c "import sys;lines=[f'- {x}' if x.startswith('Bump') else f'  - {x}' for x in sys.stdin]; print(''.join(['# Changelog\n'] + lines))" > changelog.md
+	git log --pretty="%s [%h](https://github.com/esgf-nimbus/nimbus/commit/%h) (%cs)" . | python3 -c "import sys;lines=[f'- {x}' if x.startswith('Bump') else f'  - {x}' for x in sys.stdin]; print(''.join(['# Changelog\n'] + lines))" > changelog.md
 
 bump-%:
 	tbump $(if $(COMMIT),,--dry-run) --no-tag --no-push $(shell pysemver bump $* $(shell tbump current-version)) $(if $(COMMIT),&& make changelog && git add changelog.md && git commit -m "Updates changelog",)

dockerfiles/minimal-notebook-gpu/Dockerfile → dockerfiles/climate-notebook-gpu/Dockerfile

@@ -1,5 +1,5 @@
-ARG TAG=0.1.18
-ARG IMAGE=ghcr.io/esgf-nimbus/minimal-notebook
+ARG TAG=0.1.0
+ARG IMAGE=ghcr.io/esgf-nimbus/climate-notebook
 FROM ${IMAGE}:${TAG}
 
 USER root

dockerfiles/minimal-notebook-gpu/Makefile → dockerfiles/climate-notebook-gpu/Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL = build
 
-VERSION = 0.1.6
+VERSION = 0.1.0
 
 run: ARGS = -p 8888:8888
 

dockerfiles/climate-notebook-gpu/changelog.md

@@ -0,0 +1,2 @@
+# Changelog
+

dockerfiles/minimal-notebook-gpu/tbump.toml → dockerfiles/climate-notebook-gpu/tbump.toml

@@ -1,5 +1,5 @@
 [version]
-current = "0.1.6"
+current = "0.1.0"
 
 regex = '''
   (?P<major>\d+)
@@ -10,7 +10,7 @@ regex = '''
   '''
 
 [git]
-message_template = "Bump minimal-notebook-gpu to {new_version}"
+message_template = "Bump climate-notebook-gpu to {new_version}"
 tag_template = "v{new_version}"
 
 [[file]]

dockerfiles/climate-notebook/Dockerfile

@@ -0,0 +1,39 @@
+ARG TAG=0.1.19
+FROM python:3.10 as intake-es-builder
+
+COPY intake_es /intake_es
+
+WORKDIR /intake_es
+
+RUN pip install setuptools build && \
+    python -m build
+
+FROM ghcr.io/esgf-nimbus/minimal-notebook:${TAG}
+
+LABEL org.opencontainers.image.source="https://github.com/esgf-nimbus/nimbus"
+LABEL org.opencontainers.image.url="https://github.com/esgf-nimbus/nimbus/blob/main/dockerfiles/climate-notebook/Dockerfile"
+
+USER root
+
+COPY base.yaml /base.yaml
+
+RUN --mount=type=cache,target=/opt/conda/pkgs \
+    mamba env update -n base -f /base.yaml && \
+    rm /base.yaml && \
+    pip install ecoport && \
+    python -m ecoport -f /base.yaml
+
+COPY --from=intake-es-builder /intake_es/dist /build
+
+RUN python -m pip install /build/*.tar.gz && \
+    rm -rf /build
+
+COPY scripts /scripts
+COPY notebooks /notebooks
+COPY changelog.md /changelog.md
+
+RUN source /scripts/render-intro.sh
+
+USER $NB_USER
+
+ENTRYPOINT ["/scripts/entrypoint.sh"]

dockerfiles/climate-notebook/Makefile

@@ -0,0 +1,14 @@
+.DEFAULT_GOAL = build
+
+VERSION = 0.1.0
+
+run: ARGS = -p 8888:8888 -v $(PWD)/../dask-gateway/conda-envs:/conda-envs
+
+build: POST_BUILD = make export-env
+
+.PHONY: export-env
+export-env: ENTRYPOINT = /bin/bash
+export-env: COMMAND = -c "cp /base.yaml /conda-envs/"
+export-env: run
+
+include ../Makefile