From b3d2600db9cffbca7edbe2941e4d84274625c302 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 11 Jun 2024 10:23:27 +0300 Subject: [PATCH 01/11] Initial commit for building a docker image for DKB --- .../workflows/build_almalinux_9_docker.yaml | 61 ++++++++++++++++++ Installation/Docker/AL9/Dockerfile | 64 +++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 .github/workflows/build_almalinux_9_docker.yaml create mode 100644 Installation/Docker/AL9/Dockerfile diff --git a/.github/workflows/build_almalinux_9_docker.yaml b/.github/workflows/build_almalinux_9_docker.yaml new file mode 100644 index 000000000..747159aff --- /dev/null +++ b/.github/workflows/build_almalinux_9_docker.yaml @@ -0,0 +1,61 @@ +# +name: Create and publish a Docker DKB image for AlmaLinux9 + +# Configures this workflow to run every time a change is pushed to the branch called `docker_build`. +on: + push: + branches: ['docker_build'] + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.BUILD_DOCKER_GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + + diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile new file mode 100644 index 000000000..ddc13b1dc --- /dev/null +++ b/Installation/Docker/AL9/Dockerfile @@ -0,0 +1,64 @@ +# @project The Data Knowledge Base (DKB) +# @copyright Copyright © 2024 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +# DKB generic image + +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base + +# DKB environment variables +ENV DKB_DIR="/opt/dkb3" \ + DKB_GIT="https://github.com/PanDAWMS/dkb.git" + +RUN yum install -y \ + git \ + wget \ + diffutils + +# Prepare DKB software +RUN mkdir ${DKB_DIR} \ + && \ + cd ${DKB_DIR} \ + && \ + git clone ${DKB_GIT} dkb.git \ + && \ + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && \ + sh ./Miniconda3-latest-Linux-x86_64.sh -b -p /opt/dkb3/miniconda3 -s -m \ + && \ + source "/opt/dkb3/miniconda3/etc/profile.d/conda.sh" \ + && \ + conda create -y -n dkb_python_3_9 python=3.9 \ + && \ + conda activate dkb_python_3_9 \ + && \ + conda install -y Elasticsearch cchardet cchardet cx_Oracle \ + && \ + pip install pyAMI_core \ + && \ + pip install pyAMI_atlas \ + && \ + sed -e "s/ input(prompt)/'y'/" -i /opt/dkb3/miniconda3/envs/dkb_python_3_9/bin/ami_atlas_post_install \ + && \ + ami_atlas_post_install \ + && \ + pip install rucio \ + && \ + cd dkb.git/Utils/Dataflow/test/pyDKB \ + && \ + ./test.sh + +# Cleanup image +RUN yum clean all ; \ + rm -rf ${DKB_DIR}/Miniconda3-latest-Linux-x86_64.sh From 4eb08d3f9d8179dd2a431eee6a96cbe01cf09f79 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 11 Jun 2024 10:40:44 +0300 Subject: [PATCH 02/11] Change path for Dockerfile --- .github/workflows/build_almalinux_9_docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_almalinux_9_docker.yaml b/.github/workflows/build_almalinux_9_docker.yaml index 747159aff..12a4366de 100644 --- a/.github/workflows/build_almalinux_9_docker.yaml +++ b/.github/workflows/build_almalinux_9_docker.yaml @@ -49,6 +49,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + dockerfile: Installation/Docker/AL9/Dockerfile # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - name: Generate artifact attestation From b47ac2a49945b2ca31091e9983b4fbf8cb67e041 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 11 Jun 2024 10:45:19 +0300 Subject: [PATCH 03/11] Change input parameter for dockerfile --- .github/workflows/build_almalinux_9_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_almalinux_9_docker.yaml b/.github/workflows/build_almalinux_9_docker.yaml index 12a4366de..340f77899 100644 --- a/.github/workflows/build_almalinux_9_docker.yaml +++ b/.github/workflows/build_almalinux_9_docker.yaml @@ -49,7 +49,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - dockerfile: Installation/Docker/AL9/Dockerfile + file: Installation/Docker/AL9/Dockerfile # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." - name: Generate artifact attestation From 906ed36f306f6b25730dfee7a4bcb6e916443412 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 11 Jun 2024 11:52:10 +0300 Subject: [PATCH 04/11] Try GITHUB_TOKEN --- .github/workflows/build_almalinux_9_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_almalinux_9_docker.yaml b/.github/workflows/build_almalinux_9_docker.yaml index 340f77899..3e12c2c12 100644 --- a/.github/workflows/build_almalinux_9_docker.yaml +++ b/.github/workflows/build_almalinux_9_docker.yaml @@ -31,7 +31,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.BUILD_DOCKER_GITHUB_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta From 32eab4742093f8dce1660d8361e251d077e73c71 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Thu, 13 Jun 2024 08:55:27 +0300 Subject: [PATCH 05/11] Add pytz to conda env --- Installation/Docker/AL9/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index ddc13b1dc..00e9f6d06 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -43,7 +43,7 @@ RUN mkdir ${DKB_DIR} \ && \ conda activate dkb_python_3_9 \ && \ - conda install -y Elasticsearch cchardet cchardet cx_Oracle \ + conda install -y Elasticsearch cchardet cchardet cx_Oracle pytz\ && \ pip install pyAMI_core \ && \ From d5947bac05dc5a19aff19e0662c1038f7dc69de2 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Thu, 13 Jun 2024 10:02:02 +0300 Subject: [PATCH 06/11] Add necessary packages for consistency check --- Installation/Docker/AL9/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index 00e9f6d06..4b5df3556 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -24,7 +24,9 @@ ENV DKB_DIR="/opt/dkb3" \ RUN yum install -y \ git \ wget \ - diffutils + diffutils \ + libnsl \ + libaio # Prepare DKB software RUN mkdir ${DKB_DIR} \ From 7a87206b38b11d1854f62d7be0811af647734b9c Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 18 Jun 2024 16:16:03 +0300 Subject: [PATCH 07/11] Add fix for pyAMI python code --- Installation/Docker/AL9/Dockerfile | 3 + Installation/Docker/AL9/httpclient.py | 174 ++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 Installation/Docker/AL9/httpclient.py diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index 4b5df3556..7443298b7 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -61,6 +61,9 @@ RUN mkdir ${DKB_DIR} \ && \ ./test.sh +# Patched version of pyAMI +COPY httpclient.py /opt/dkb3/miniconda3/envs/dkb_python_3_9/lib/python3.9/site-packages/pyAMI/httpclient.py + # Cleanup image RUN yum clean all ; \ rm -rf ${DKB_DIR}/Miniconda3-latest-Linux-x86_64.sh diff --git a/Installation/Docker/AL9/httpclient.py b/Installation/Docker/AL9/httpclient.py new file mode 100644 index 000000000..1f02bfcaf --- /dev/null +++ b/Installation/Docker/AL9/httpclient.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +from __future__ import (division, print_function, unicode_literals) +############################################################################# +# Author : Jerome ODIER, Jerome FULACHIER, Fabian LAMBERT, Solveig ALBRAND +# +# Email : jerome.odier@lpsc.in2p3.fr +# jerome.fulachier@lpsc.in2p3.fr +# fabian.lambert@lpsc.in2p3.fr +# solveig.albrand@lpsc.in2p3.fr +# +# Version : 5.X.X (2014) +# +############################################################################# + +import ssl, sys, pyAMI.config, pyAMI.exception + +if sys.version_info[0] == 3: + import http.client as http_client +else: + import httplib as http_client + +############################################################################# + +headers = { + 'Accept': 'text/plain', + 'User-Agent': 'pyAMI/%s' % pyAMI.config.version, + 'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8', +} + +############################################################################# + +class HttpClient(object): + ##################################################################### + + def __init__(self, config): + self.config = config + + self.endpoint = None + self.connection = None + + ##################################################################### + + def create_unverified_context(slef): + + result = ssl.SSLContext(ssl.PROTOCOL_TLS) + + result.options |= ssl.OP_NO_SSLv2 + result.options |= ssl.OP_NO_SSLv3 + + result.check_hostname = False + + return result + + ##################################################################### + + def connect(self, endpoint): + ############################################################# + # SET ENDPOINT # + ############################################################# + + self.endpoint = endpoint + + ############################################################# + # HTTP CONNECTION # + ############################################################# + + if self.endpoint['prot'] == 'http': + + self.connection = http_client.HTTPConnection( + str(self.endpoint['host']), + int(self.endpoint['port']) + ) + + ############################################################# + # HTTPS CONNECTION # + ############################################################# + + elif self.endpoint['prot'] == 'https': + + if self.config.conn_mode == self.config.CONN_MODE_LOGIN: + ############################################# + # WITHOUT CERTIFICATE # + ############################################# + + try: + context = self.create_unverified_context() + + self.connection = http_client.HTTPSConnection( + str(self.endpoint['host']), + int(self.endpoint['port']), + key_file = None, + cert_file = None, + context = context + ) + + except AttributeError as e: + + self.connection = http_client.HTTPSConnection( + str(self.endpoint['host']), + int(self.endpoint['port']), + key_file = None, + cert_file = None + ) + + else: + ############################################# + # WITH CERTIFICATE # + ############################################# + + try: + context = self.create_unverified_context() + + self.connection = http_client.HTTPSConnection( + str(self.endpoint['host']), + int(self.endpoint['port']), + key_file = self.config.key_file, + cert_file = self.config.cert_file, + context = context + ) + + except AttributeError as e: + + self.connection = http_client.HTTPSConnection( + str(self.endpoint['host']), + int(self.endpoint['port']), + key_file = self.config.key_file, + cert_file = self.config.cert_file + ) + + ############################################################# + + else: + raise pyAMI.exception.Error('invalid endpoint protocol `%s`, not in [http, https]' % self.endpoint['prot']) + + ##################################################################### + + def close(self): + self.connection.close() + + ##################################################################### + + def request(self, data): + ############################################################# + # DO REQUEST # + ############################################################# + + headers['Cookie'] = self.config.jsid + + try: + self.connection.request('POST', self.endpoint['path'], data, headers) + + except Exception as e: + raise pyAMI.exception.Error('could not connect to `%s://%s:%s%s`: %s' % ( + self.endpoint['prot'], + self.endpoint['host'], + self.endpoint['port'], + self.endpoint['path'], + e + )) + + ############################################################# + # GET RESPONSE AND COOKIE # + ############################################################# + + result = self.connection.getresponse() + + self.config.jsid = result.getheader('set-cookie').replace('%', '%%') + + + ############################################################# + + return result + +############################################################################# From 485e8fdbf3525422baf65f472888b0404a4aa1a2 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 18 Jun 2024 16:19:59 +0300 Subject: [PATCH 08/11] Change path for fixed file --- Installation/Docker/AL9/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index 7443298b7..5411df559 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -62,7 +62,7 @@ RUN mkdir ${DKB_DIR} \ ./test.sh # Patched version of pyAMI -COPY httpclient.py /opt/dkb3/miniconda3/envs/dkb_python_3_9/lib/python3.9/site-packages/pyAMI/httpclient.py +COPY ./httpclient.py /opt/dkb3/miniconda3/envs/dkb_python_3_9/lib/python3.9/site-packages/pyAMI/httpclient.py # Cleanup image RUN yum clean all ; \ From 6709c335295c7a4ed453c3469e2e7eef9d884d72 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Tue, 18 Jun 2024 16:23:44 +0300 Subject: [PATCH 09/11] Fix2 for dir --- Installation/Docker/AL9/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index 5411df559..154ed81b3 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -62,7 +62,7 @@ RUN mkdir ${DKB_DIR} \ ./test.sh # Patched version of pyAMI -COPY ./httpclient.py /opt/dkb3/miniconda3/envs/dkb_python_3_9/lib/python3.9/site-packages/pyAMI/httpclient.py +COPY Installation/Docker/AL9/httpclient.py /opt/dkb3/miniconda3/envs/dkb_python_3_9/lib/python3.9/site-packages/pyAMI/httpclient.py # Cleanup image RUN yum clean all ; \ From 2b17b9a5728dcfef5e0c348d8019b5d3559722d9 Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Wed, 19 Jun 2024 11:56:26 +0300 Subject: [PATCH 10/11] Add php packages for dkb --- Installation/Docker/AL9/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Installation/Docker/AL9/Dockerfile b/Installation/Docker/AL9/Dockerfile index 154ed81b3..7e7dd2946 100644 --- a/Installation/Docker/AL9/Dockerfile +++ b/Installation/Docker/AL9/Dockerfile @@ -26,7 +26,9 @@ RUN yum install -y \ wget \ diffutils \ libnsl \ - libaio + libaio \ + php \ + php-cli # Prepare DKB software RUN mkdir ${DKB_DIR} \ From 5cda34b079f606d509833568dd3d3304c370434d Mon Sep 17 00:00:00 2001 From: Victor Kotlyar Date: Thu, 11 Jul 2024 15:53:08 +0300 Subject: [PATCH 11/11] Change for build docker image on any branc push --- .github/workflows/build_almalinux_9_docker.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_almalinux_9_docker.yaml b/.github/workflows/build_almalinux_9_docker.yaml index 3e12c2c12..634cc5925 100644 --- a/.github/workflows/build_almalinux_9_docker.yaml +++ b/.github/workflows/build_almalinux_9_docker.yaml @@ -1,10 +1,8 @@ # name: Create and publish a Docker DKB image for AlmaLinux9 -# Configures this workflow to run every time a change is pushed to the branch called `docker_build`. -on: - push: - branches: ['docker_build'] +# Configures this workflow to run every time a change is pushed to the any branch. +on: push # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: