From afda625052023c14c85128842c124528b385f9f6 Mon Sep 17 00:00:00 2001 From: dd84ai Date: Fri, 10 Jan 2025 11:26:18 +0100 Subject: [PATCH] refactor: ci code to be from kcl --- .github/workflows/build.yml | 73 ------------------------- .github/workflows/docker-production.yml | 54 ++++++++++++++++++ .github/workflows/docker-staging.yml | 53 ++++++++++++++++++ .github/workflows/pages.yml | 50 ++++++++--------- .github/workflows/test.yml | 35 +++++------- Taskfile.yml | 4 ++ kcls/build.k | 15 +++++ kcls/common.k | 1 + kcls/kcl.mod | 7 +++ kcls/kcl.mod.lock | 5 ++ kcls/wf_docker_production.k | 38 +++++++++++++ kcls/wf_docker_staging.k | 41 ++++++++++++++ kcls/wg_pages.k | 46 ++++++++++++++++ kcls/wg_test.k | 33 +++++++++++ 14 files changed, 336 insertions(+), 119 deletions(-) delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/docker-production.yml create mode 100644 .github/workflows/docker-staging.yml create mode 100644 kcls/build.k create mode 100644 kcls/common.k create mode 100644 kcls/kcl.mod create mode 100644 kcls/kcl.mod.lock create mode 100644 kcls/wf_docker_production.k create mode 100644 kcls/wf_docker_staging.k create mode 100644 kcls/wg_pages.k create mode 100644 kcls/wg_test.k diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 8cf12e2..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Build - -on: - workflow_dispatch: - push: - tags: - - "*" - -env: - DOCKER_HOST: ssh://root@darkbot - -jobs: - build: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - - name: Install Taskfile - uses: darklab8/infra/.github/actions/install-taskfile@master - - - name: Set up Go - uses: darklab8/infra/.github/actions/install-go@master - - # SOURCE_NAME: The branch or the tag - # SOURCE_BRANCH: The branch or empty - # SOURCE_TAG: The tag or empty - - name: Find tags - id: data - run: | - echo "BUILD_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - echo "SOURCE_NAME=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - echo "SOURCE_BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT - echo "SOURCE_TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - - - - name: Echo tags - run: | - echo ${{ steps.data.outputs.BUILD_VERSION }} - - - name: Docker login - run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login --username darkwind8 --password-stdin - - - name: Install ssh key - run: | - mkdir ~/.ssh | true - echo "${{ secrets.SSH_KEY_BASE64 }}" | base64 --decode > ~/.ssh/id_rsa - echo "Host darkbot" > ~/.ssh/config - echo " HostName 37.27.207.42" >> ~/.ssh/config - echo " User root" >> ~/.ssh/config - echo " IdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config - echo " IdentitiesOnly yes" >> ~/.ssh/config - chmod 700 ~/.ssh - chmod 600 ~/.ssh/* - ssh-keyscan -H 37.27.207.42 >> ~/.ssh/known_hosts | true - ssh-keyscan -H darkbot >> ~/.ssh/known_hosts | true - - - name: Build image - run: docker build --build-arg "BUILD_VERSION=${{ steps.data.outputs.BUILD_VERSION }}" --tag darkwind8/darkbot:${{ steps.data.outputs.BUILD_VERSION }} . - - name: Push image - run: | - docker push darkwind8/darkbot:${{ steps.data.outputs.BUILD_VERSION }} - docker tag darkwind8/darkbot:${{ steps.data.outputs.BUILD_VERSION }} darkwind8/darkbot:staging - docker push darkwind8/darkbot:staging - - - name: Switch service image to new one - run: | - docker pull darkwind8/darkbot:staging && docker service update --image darkwind8/darkbot:staging darkbot-staging - - # this stuff is for Docker mode only - # - name: Install arm64 redock at server (not amd64 in my case) - # run: ssh root@darkbot 'curl -L $(curl -Ls -o /dev/null -w %{url_effective} https://github.com/darklab8/redock/releases/latest | sed "s/releases\/tag/releases\/download/")/redock-linux-arm64 -o /usr/local/bin/redock && chmod 777 /usr/local/bin/redock' - # - name: Run redock # for docker mod only needed - # run: ssh root@darkbot 'redock --ctr=darkbot-staging --strict_pull=true --image_name=darkwind8/darkbot:${{ steps.data.outputs.BUILD_VERSION }}' diff --git a/.github/workflows/docker-production.yml b/.github/workflows/docker-production.yml new file mode 100644 index 0000000..2531b0c --- /dev/null +++ b/.github/workflows/docker-production.yml @@ -0,0 +1,54 @@ +name: Deploy docker Production +'on': + push: + tags: + - '*' + - '!*-*' + workflow_dispatch: {} +jobs: + job: + name: Deploy docker Production + runs-on: ubuntu-22.04 + steps: + - name: Check out source repository + uses: actions/checkout@v3 + - name: Docker login + run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login --username darkwind8 --password-stdin + - name: Install Darklab ssh key + run: | + mkdir ~/.ssh | true + echo "${{ secrets.SSH_KEY_BASE64 }}" | base64 --decode > ~/.ssh/id_rsa + echo "Host darkbot" > ~/.ssh/config + echo " HostName 37.27.207.42" >> ~/.ssh/config + echo " User root" >> ~/.ssh/config + echo " IdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config + echo " IdentitiesOnly yes" >> ~/.ssh/config + chmod 700 ~/.ssh + chmod 600 ~/.ssh/* + ssh-keyscan -H 37.27.207.42 >> ~/.ssh/known_hosts | true + ssh-keyscan -H darkbot >> ~/.ssh/known_hosts | true + - name: Get version from tag + run: | + set -x + echo "BUILD_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + id: version + - name: Build image darkwind8/darkbot + run: | + set -ex + export tag_version=${{ steps.version.outputs.BUILD_VERSION }} + docker build --build-arg "BUILD_VERSION=$tag_version" --tag darkwind8/darkbot:$tag_version . + + docker tag darkwind8/darkbot:$tag_version darkwind8/darkbot:production + + docker push darkwind8/darkbot:$tag_version + sleep 5 + docker push darkwind8/darkbot:production + sleep 5 + - name: Switch service image darkwind8/darkbot for swarm to new one + run: | + set -ex + docker pull darkwind8/darkbot:production + sleep 5 + docker service update --image darkwind8/darkbot:production darkbot-production +env: + DOCKER_HOST: ssh://root@darkbot diff --git a/.github/workflows/docker-staging.yml b/.github/workflows/docker-staging.yml new file mode 100644 index 0000000..420cfea --- /dev/null +++ b/.github/workflows/docker-staging.yml @@ -0,0 +1,53 @@ +name: Deploy docker staging +'on': + push: + tags: + - '*-*' + workflow_dispatch: {} +jobs: + job: + name: Deploy docker staging + runs-on: ubuntu-22.04 + steps: + - name: Check out source repository + uses: actions/checkout@v3 + - name: Get version from tag + run: | + set -x + echo "BUILD_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + id: version + - name: Docker login + run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login --username darkwind8 --password-stdin + - name: Install Darklab ssh key + run: | + mkdir ~/.ssh | true + echo "${{ secrets.SSH_KEY_BASE64 }}" | base64 --decode > ~/.ssh/id_rsa + echo "Host darkbot" > ~/.ssh/config + echo " HostName 37.27.207.42" >> ~/.ssh/config + echo " User root" >> ~/.ssh/config + echo " IdentityFile ~/.ssh/id_rsa" >> ~/.ssh/config + echo " IdentitiesOnly yes" >> ~/.ssh/config + chmod 700 ~/.ssh + chmod 600 ~/.ssh/* + ssh-keyscan -H 37.27.207.42 >> ~/.ssh/known_hosts | true + ssh-keyscan -H darkbot >> ~/.ssh/known_hosts | true + - name: Build image darkwind8/darkbot + run: | + set -ex + export tag_version=${{ steps.version.outputs.BUILD_VERSION }} + docker build --build-arg "BUILD_VERSION=$tag_version" --tag darkwind8/darkbot:$tag_version . + + docker tag darkwind8/darkbot:$tag_version darkwind8/darkbot:staging + + docker push darkwind8/darkbot:$tag_version + sleep 5 + docker push darkwind8/darkbot:staging + sleep 5 + - name: Switch service image darkwind8/darkbot for swarm to new one + run: | + set -ex + docker pull darkwind8/darkbot:staging + sleep 5 + docker service update --image darkwind8/darkbot:staging darkbot-staging +env: + DOCKER_HOST: ssh://root@darkbot diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 89cd444..c988379 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -1,37 +1,35 @@ name: Pages - -on: +'on': push: - branches: ["master"] - + branches: + - master + workflow_dispatch: {} jobs: - github-pages: - name: Github Pages + job: + name: Pages runs-on: ubuntu-22.04 permissions: pages: write id-token: write + contents: read environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - - name: Check out source repository - uses: actions/checkout@v2 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Ensure pip - run: > - python3 --version - python3 -m ensurepip - - name: Install requirements.txt - run: pip install -r requirements.txt -c constraints.txt - - name: Build docs - run: mkdocs build - - uses: actions/upload-pages-artifact@v1 - with: - name: github-pages - path: ./site - - name: Deploy to pages - uses: actions/deploy-pages@v1 - id: deployment + - name: Check out source repository + uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: ensure pip + run: python3 -m ensurepip + - name: Install requirements.txt + run: pip install -r requirements.txt -c constraints.txt + - name: Build docs + run: mkdocs build + - uses: actions/upload-pages-artifact@v1 + with: + name: github-pages + path: './site' + - uses: actions/deploy-pages@v1 + id: deployment diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7dac124..53502cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,26 +1,21 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - name: Test - -on: +'on': push: - branches: ["master"] + branches: + - master schedule: - # every sunday - - cron: '0 10 1 * *' - + - cron: '0 10 1 * *' + workflow_dispatch: {} jobs: - test: + job: + name: Test runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 - - - name: Install Taskfile - uses: darklab8/infra/.github/actions/install-taskfile@master - - - name: Set up Go - uses: darklab8/infra/.github/actions/install-go@master - - - name: Test - run: task test + - name: Check out source repository + uses: actions/checkout@v3 + - name: Install Go + uses: darklab8/infra/.github/actions/install-go@master + - name: Install Taskfile + uses: darklab8/infra/.github/actions/install-taskfile@master + - name: test + run: task test diff --git a/Taskfile.yml b/Taskfile.yml index 6e387c1..e14927b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -130,3 +130,7 @@ tasks: cmds: - task: profile:render - task: profile:render2 + + kcl:build: + cmds: + - kcl kcls/build.k diff --git a/kcls/build.k b/kcls/build.k new file mode 100644 index 0000000..8c1b18a --- /dev/null +++ b/kcls/build.k @@ -0,0 +1,15 @@ +import .wf_docker_staging +import .wf_docker_production +import .wg_test +import .wg_pages + +import yaml + +_workflows = [ + wf_docker_staging._workflow + wf_docker_production._workflow + wg_test._workflow + wg_pages._workflow +] + +_run = [yaml.dump_to_file(a, ".github/workflows/${a._filename}", ignore_private=True, ignore_none=True) for a in _workflows] diff --git a/kcls/common.k b/kcls/common.k new file mode 100644 index 0000000..b9d0b58 --- /dev/null +++ b/kcls/common.k @@ -0,0 +1 @@ +_image_name = "darkwind8/darkbot" diff --git a/kcls/kcl.mod b/kcls/kcl.mod new file mode 100644 index 0000000..b0518a9 --- /dev/null +++ b/kcls/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "kcls" +edition = "v0.11.0" +version = "0.0.1" + +[dependencies] +infra = { path = "../../infra", version = "0.0.1" } diff --git a/kcls/kcl.mod.lock b/kcls/kcl.mod.lock new file mode 100644 index 0000000..1faec47 --- /dev/null +++ b/kcls/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.infra] + name = "infra" + full_name = "infra_0.0.1" + version = "0.0.1" diff --git a/kcls/wf_docker_production.k b/kcls/wf_docker_production.k new file mode 100644 index 0000000..92ccf66 --- /dev/null +++ b/kcls/wf_docker_production.k @@ -0,0 +1,38 @@ +import infra.kcls.models.github_workflow as gw +import infra.kcls.common.github as ghc +import infra.kcls.common.docker +import .common as c + +_workflow_name = "Deploy docker Production" +_workflow = gw.Workflow { + _filename = "docker-production.yml" + name = _workflow_name + on = { + workflow_dispatch: {} + push.tags: ghc.TagsSemverProduction + } + env = ghc.DockerHostEnv + jobs = { + job = _production_deploy_job + } +} +_production_deploy_job: gw.Job = { + name = _workflow_name + steps = [ + ghc.CheckoutRepo + ghc.DockerLogin + ghc.InstallDarklabSshKey + ghc.GetVersionFromTag + docker.Build(docker.BuildOpts { + image_name = c._image_name + tag = ghc.GetVersionOutput + extra_tags = ["production"] + push = True + }) + docker.DeploySwarm(docker.DeploySwarmOpts { + image_name = c._image_name + tag = "production" + service_name = "darkbot-production" + }) + ] +} diff --git a/kcls/wf_docker_staging.k b/kcls/wf_docker_staging.k new file mode 100644 index 0000000..97f745f --- /dev/null +++ b/kcls/wf_docker_staging.k @@ -0,0 +1,41 @@ +import infra.kcls.models.github_workflow as gw +import infra.kcls.common.github as ghc +import infra.kcls.common.docker +import infra.kcls.actions.install_go as go +import infra.kcls.actions.install_taskfile as tf +import infra.kcls.actions.install_autogit as autogit +import .common as c + +_workflow_name = "Deploy docker staging" +_workflow = gw.Workflow { + _filename = "docker-staging.yml" + name = _workflow_name + on = { + push.tags: ghc.TagsSemverStaging + workflow_dispatch: {} + } + env = ghc.DockerHostEnv + jobs = { + job = _test_job + } +} +_test_job: gw.Job = { + name = _workflow_name + steps = [ + ghc.CheckoutRepo + ghc.GetVersionFromTag + ghc.DockerLogin + ghc.InstallDarklabSshKey + docker.Build(docker.BuildOpts { + image_name = c._image_name + tag = ghc.GetVersionOutput + extra_tags = ["staging"] + push = True + }) + docker.DeploySwarm(docker.DeploySwarmOpts { + image_name = c._image_name + tag = "staging" + service_name = "darkbot-staging" + }) + ] +} diff --git a/kcls/wg_pages.k b/kcls/wg_pages.k new file mode 100644 index 0000000..ab0d32b --- /dev/null +++ b/kcls/wg_pages.k @@ -0,0 +1,46 @@ +import infra.kcls.models.github_workflow as gw +import infra.kcls.common.github as ghc +import infra.kcls.actions.install_go as go +import infra.kcls.actions.install_taskfile as tf +import infra.kcls.models.github_action as ga + +_workflow_name = "Pages" +_workflow = gw.Workflow { + _filename = "pages.yml" + name = _workflow_name + on = { + workflow_dispatch: {} + push = { + branches = ["master"] + } + } + jobs = { + job = _pages_job + } +} +_pages_job: gw.Job = { + name = _workflow_name + permissions: ghc.PagesPermissions {} + environment: dict(ghc.PagesEnvironment {}) + steps = [ + ghc.CheckoutRepo + ga.Step { + uses = "actions/setup-python@v4" + with = {"python-version": "3.10"} + } + ga.Step { + name = "ensure pip" + run = "python3 -m ensurepip" + } + ga.Step { + name = "Install requirements.txt" + run = "pip install -r requirements.txt -c constraints.txt" + } + ga.Step { + name = "Build docs" + run = "mkdocs build" + } + ] + ghc.PagesToDeploySteps(ghc.PagesDeployOpts { + path = "./site" + }) +} diff --git a/kcls/wg_test.k b/kcls/wg_test.k new file mode 100644 index 0000000..dfa4a73 --- /dev/null +++ b/kcls/wg_test.k @@ -0,0 +1,33 @@ +import infra.kcls.models.github_workflow as gw +import infra.kcls.common.github as ghc +import infra.kcls.actions.install_go as go +import infra.kcls.actions.install_taskfile as tf +import infra.kcls.models.github_action as ga + +_workflow_name = "Test" +_workflow = gw.Workflow { + _filename = "test.yml" + name = _workflow_name + on = { + workflow_dispatch: {} + push = { + branches = ["master"] + } + schedule: [{cron = '0 10 1 * *'}] + } + jobs = { + job = _test_job + } +} +_test_job: gw.Job = { + name = _workflow_name + steps = [ + ghc.CheckoutRepo + go.InstallGo {} + tf.InstallTaskfile {} + ga.Step { + name = "test" + run: "task test" + } + ] +}