.gitlab-ci.yml

default:
  image: docker.io/zazukoians/node-java-jena:v5

stages:
  - github
  - fetch
  - process
  - store

# Push `main` branch to GitHub
github:
  stage: github
  image:
    name: docker.io/bitnami/git:2
  script:
    # Some SSH configuration
    - mkdir -p /root/.ssh
    - ssh-keyscan -H github.com >> /root/.ssh/known_hosts
    - cp "${GITHUB_REPO_PRIVATE_KEY}" /root/.ssh/id_ed25519
    - chmod 400 /root/.ssh/id_ed25519

    # Create the main branch locally out of the detached HEAD
    - git checkout -b main

    # Inspect the current Git state
    - git branch --list
    - git log --oneline

    # Add the remote GitHub repository, and push the current branch to it
    - git remote add github "${GITHUB_REPO}"
    - git push --follow-tags --force github main
  only:
    - main

# fetch the latest full import from the S3 bucket
fetch-test:
  stage: fetch
  interruptible: true
  image:
    name: docker.io/bitnami/minio-client:latest
  artifacts:
    expire_in: 1 day
    paths:
      - input/*.json
  script:
    - mc alias set bucket "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
    - mkdir -p input
    - mc ls "bucket/${S3_BUCKET}/${S3_PATH}/"
    - ./scripts/fetch_input.sh
  except:
    - main
fetch-prod:
  stage: fetch
  interruptible: true
  image:
    name: docker.io/bitnami/minio-client:latest
  artifacts:
    expire_in: 1 day
    paths:
      - input/*.json
  script:
    - mc alias set bucket "${S3_ENDPOINT}" "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"
    - mkdir -p input
    - mc ls "bucket/${S3_BUCKET}/${S3_PATH}/"
    - ./scripts/fetch_input.sh
  environment:
    name: prod
  only:
    - main

# Install npm dependencies
npm-deps:
  stage: fetch
  timeout: 5 minutes
  image: docker.io/library/node:22
  retry: 2
  artifacts:
    expire_in: 1 day
    paths:
      - node_modules/
  script:
    - npm ci

# generate metadata as ntriples
metadata:
  stage: process
  artifacts:
    expire_in: 1 week
    paths:
      - output/metadata.nt
  script:
    - mkdir -p output
    # ignore the exit code (as warnings are making the command fail)
    - riot --output=ntriples metadata/*.ttl > output/metadata.nt || true
    # check the existance of the file
    - "[ -f output/metadata.nt ]"
    # make sure it has more than one line
    - '[ "$(wc -l output/metadata.nt | awk ''{ print $1 }'')" -gt 1 ]'

# run the pipeline on each splitted part that are generated within this step
process-test:
  stage: process
  interruptible: true
  dependencies:
    - npm-deps
    - fetch-test
  artifacts:
    expire_in: 1 week
    paths:
      - output/*.nt
  services:
    - name: ghcr.io/zazuko/carml-service:v0.0.5
      alias: carml-service
  variables:
    CARML_SERVICE: "http://carml-service:8080/"
  script:
    - ./scripts/splitter.sh
    - ./scripts/generate_triples.sh
  except:
    - main
process-prod:
  stage: process
  interruptible: true
  dependencies:
    - npm-deps
    - fetch-prod
  artifacts:
    expire_in: 1 week
    paths:
      - output/*.nt
  services:
    - name: ghcr.io/zazuko/carml-service:v0.0.5
      alias: carml-service
  variables:
    CARML_SERVICE: "http://carml-service:8080/"
  script:
    - ./scripts/splitter.sh
    - ./scripts/generate_triples.sh
  environment:
    name: prod
  only:
    - main

# store triples in the triplestore
store-test:
  stage: store
  resource_group: store-test
  interruptible: true
  dependencies:
    - metadata
    - process-test
  script:
    # display the list of all files that we will upload in the triplestore
    - ls -alh output
    # upload them to the triplestore
    - ./scripts/upload.sh
  only:
    - develop
store-prod:
  stage: store
  resource_group: store-prod
  interruptible: true
  dependencies:
    - metadata
    - process-prod
  script:
    # display the list of all files that we will upload in the triplestore
    - ls -alh output
    # upload them to the triplestore
    - ./scripts/upload.sh
  environment:
    name: prod
  only:
    - main