From bcaeed722501d111087f8a4cfc57dae05ce46945 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 6 Jan 2022 08:32:32 -0600 Subject: [PATCH] allow the driver to run in dummy mode (#382) The driver should fall back to dummy mode when dependencies are not installed. This will be useful in situations where the driver deps are not installed in client applications. Driver tests have been moved into separate files allowing the tests to be run / and rerun without conflict. Signed-off-by: Kevin Carter --- ...ull-request-functional-driver-defaults.yml | 323 +++++++ ...ll-request-functional-driver-messaging.yml | 156 ++++ .../pull-request-functional-driver-zmq.yml | 120 +++ .../pull-request-functional-operations.yml | 233 +++++ .github/workflows/pull-request-functional.yml | 827 ------------------ directord/drivers/__init__.py | 4 +- directord/interface.py | 24 +- 7 files changed, 853 insertions(+), 834 deletions(-) create mode 100644 .github/workflows/pull-request-functional-driver-defaults.yml create mode 100644 .github/workflows/pull-request-functional-driver-messaging.yml create mode 100644 .github/workflows/pull-request-functional-driver-zmq.yml create mode 100644 .github/workflows/pull-request-functional-operations.yml delete mode 100644 .github/workflows/pull-request-functional.yml diff --git a/.github/workflows/pull-request-functional-driver-defaults.yml b/.github/workflows/pull-request-functional-driver-defaults.yml new file mode 100644 index 00000000..274096bc --- /dev/null +++ b/.github/workflows/pull-request-functional-driver-defaults.yml @@ -0,0 +1,323 @@ +name: Run pull-request functional workflows defaults +on: [pull_request] +jobs: + functional_defaults_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-test-logs + path: /tmp/directord* + + functional_defaults_query_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-query.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-query-test-logs + path: /tmp/directord* + + functional_defaults_queuesentinel_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo /opt/directord/bin/directord orchestrate functional-tests-queue-sentinel.yaml + sudo timeout 240 /opt/directord/bin/directord exec --verb QUEUESENTINEL true \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-queuesentinel-test-logs + path: /tmp/directord* + + functional_defaults_async_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute async-functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async.yaml \ + --poll \ + --check + - name: Execute functional async-race-condition check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async-race.yaml \ + --poll \ + --check + - name: Execute functional async-time check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 40 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async-time.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-async-test-logs + path: /tmp/directord* + + functional_defaults_add_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async-transfer.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-add-test-logs + path: /tmp/directord* + + functional_defaults_realworld_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + {functional-tests-async-race.yaml,functional-tests-async-time.yaml,functional-tests-async.yaml,functional-tests-query.yaml,functional-tests-async-transfer.yaml} \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-realworld-test-logs + path: /tmp/directord* diff --git a/.github/workflows/pull-request-functional-driver-messaging.yml b/.github/workflows/pull-request-functional-driver-messaging.yml new file mode 100644 index 00000000..6b227373 --- /dev/null +++ b/.github/workflows/pull-request-functional-driver-messaging.yml @@ -0,0 +1,156 @@ +name: Run pull-request functional workflows messaging +on: [pull_request] +jobs: + functional_messaging_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo DRIVER=messaging bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Setup SSL encryption for Qpid + run: sudo tools/scripts/messaging/messaging-ssl-setup.sh + - name: Configure messaging_address + run: | + sudo /opt/directord/bin/python < /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo cp -R /usr/local/share/ca-certificates/directord /tmp/directord/ca-certificates + sudo cp -R /etc/qpid-dispatch /tmp/directord/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-messaging-test-logs + path: /tmp/directord* + + functional_messaging_async_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo DRIVER=messaging bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Setup SSL encryption for Qpid + run: sudo tools/scripts/messaging/messaging-ssl-setup.sh + - name: Configure messaging_address + run: | + sudo /opt/directord/bin/python < /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo cp -R /usr/local/share/ca-certificates/directord /tmp/directord/ca-certificates + sudo cp -R /etc/qpid-dispatch /tmp/directord/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-messaging-async-test-logs + path: /tmp/directord* diff --git a/.github/workflows/pull-request-functional-driver-zmq.yml b/.github/workflows/pull-request-functional-driver-zmq.yml new file mode 100644 index 00000000..b6173c5d --- /dev/null +++ b/.github/workflows/pull-request-functional-driver-zmq.yml @@ -0,0 +1,120 @@ +name: Run pull-request functional workflows zmq +on: [pull_request] +jobs: + functional_zmq_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo DRIVER=zeromq bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-zmq-test-logs + path: /tmp/directord* + + functional_zmq_async_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo DRIVER=zeromq bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute async-functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async.yaml \ + --poll \ + --check + - name: Execute functional async-race-condition check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async-race.yaml \ + --poll \ + --check + - name: Execute functional async-time check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 40 /opt/directord/bin/directord \ + orchestrate \ + functional-tests-async-time.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-zmq-async-test-logs + path: /tmp/directord* diff --git a/.github/workflows/pull-request-functional-operations.yml b/.github/workflows/pull-request-functional-operations.yml new file mode 100644 index 00000000..254870e6 --- /dev/null +++ b/.github/workflows/pull-request-functional-operations.yml @@ -0,0 +1,233 @@ +name: Run pull-request functional workflows operations +on: [pull_request] +jobs: + functional_posix_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Configure posix backend + run: >- + echo 'datastore: file:///var/cache/directord' | sudo tee -a /etc/directord/config.yaml + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-posix-test-logs + path: /tmp/directord* + + functional_redis_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo EXTRA_DEPENDENCIES=redis bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Install Redis + run: sudo apt install -y redis + - name: Start Redis + run: sudo systemctl restart redis + - name: Configure redis backend + run: >- + echo 'datastore: redis://127.0.0.1:6379/2' | sudo tee -a /etc/directord/config.yaml + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Execute functional check + run: | + cd /opt/directord/share/directord/orchestrations + sudo timeout 240 /opt/directord/bin/directord \ + orchestrate \ + functional-tests.yaml \ + --poll \ + --check + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-redis-test-logs + path: /tmp/directord* + + functional_bootstrap_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Ensure ssh is installed + run: sudo apt update; sudo apt -y install openssh-server + - name: Check config + run: | + cat > sshd_config < + ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa + - name: Add key to auth file + run: > + cat ~/.ssh/id_rsa.pub | tee -a ~/.ssh/authorized_keys + - name: Ensure the owner of the key is correct + run: | + chmod 600 ~/.ssh/authorized_keys + chmod 700 ~/.ssh + sudo chmod -c 0755 ~/ + ls -la ~/.ssh + - name: Test SSH connection to localhost + run: > + ssh -vvv -i ~/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $(whoami)@localhost + - name: Run bootstrap test + run: | + /opt/directord/bin/directord --debug \ + bootstrap \ + --catalog tools/directord-inventory-catalog.yaml \ + --catalog tests/directord-test-catalog.yaml \ + --key-file ~/.ssh/id_rsa &> /tmp/directord-bootstrap-failure.log + - name: Run bootstrap test for clients only + run: | + /opt/directord/bin/directord --debug \ + bootstrap \ + --catalog tools/directord-inventory-catalog.yaml \ + --catalog tests/directord-test-catalog-no-srvr.yaml \ + --key-file ~/.ssh/id_rsa &> /tmp/directord-bootstrap-failure-no-srvr.log + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-bootstrap-test-logs + path: /tmp/directord* + + functional_prune_check: + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Run install + run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} + - name: Run server service install + run: | + sudo /opt/directord/bin/directord-server-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-server + - name: Run client service install + run: | + sudo /opt/directord/bin/directord-client-systemd + sudo systemctl daemon-reload + sudo systemctl restart directord-client + - name: Wait for client online + run: | + timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Stop client service + run: | + sudo systemctl stop directord-client + - name: Wait for client to be pruned + run: | + timeout 240 bash -c 'while sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' + - name: Generate log details + run: | + sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true + sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true + sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true + sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true + sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true + sudo cp -R /etc/directord /tmp/ + sudo find /tmp/directord -type d -exec chmod 0755 {} \; + sudo find /tmp/directord -type f -exec chmod 0644 {} \; + if: failure() + - name: Upload build Log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: functional-prune-logs + path: /tmp/directord* diff --git a/.github/workflows/pull-request-functional.yml b/.github/workflows/pull-request-functional.yml deleted file mode 100644 index 1e43321c..00000000 --- a/.github/workflows/pull-request-functional.yml +++ /dev/null @@ -1,827 +0,0 @@ -name: Run pull-request functional workflows -on: [pull_request] -jobs: - functional_defaults_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-test-logs - path: /tmp/directord* - - functional_messaging_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo DRIVER=messaging bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Setup SSL encryption for Qpid - run: sudo tools/scripts/messaging/messaging-ssl-setup.sh - - name: Configure messaging_address - run: | - sudo /opt/directord/bin/python < /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp /etc/directord/messaging/ssl/directord* /tmp/directord - sudo cp /usr/local/share/ca-certificates/directord/* /tmp/directord - sudo cp /etc/qpid-dispatch/qdrouterd.* /tmp/directord - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-messaging-test-logs - path: /tmp/directord* - - functional_zmq_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo DRIVER=zeromq bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-zmq-test-logs - path: /tmp/directord* - - functional_posix_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Configure posix backend - run: >- - echo 'datastore: file:///var/cache/directord' | sudo tee -a /etc/directord/config.yaml - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-posix-test-logs - path: /tmp/directord* - - functional_redis_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo EXTRA_DEPENDENCIES=redis bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Install Redis - run: sudo apt install -y redis - - name: Start Redis - run: sudo systemctl restart redis - - name: Configure redis backend - run: >- - echo 'datastore: redis://127.0.0.1:6379/2' | sudo tee -a /etc/directord/config.yaml - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-redis-test-logs - path: /tmp/directord* - - functional_defaults_query_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-query.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-query-test-logs - path: /tmp/directord* - - functional_defaults_queuesentinel_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo /opt/directord/bin/directord orchestrate functional-tests-queue-sentinel.yaml - sudo timeout 240 /opt/directord/bin/directord exec --verb QUEUESENTINEL true \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-queuesentinel-test-logs - path: /tmp/directord* - - functional_defaults_async_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute async-functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async.yaml \ - --poll \ - --check - - name: Execute functional async-race-condition check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async-race.yaml \ - --poll \ - --check - - name: Execute functional async-time check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 40 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async-time.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-async-test-logs - path: /tmp/directord* - - functional_messaging_async_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo DRIVER=messaging bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Setup SSL encryption for Qpid - run: sudo tools/scripts/messaging/messaging-ssl-setup.sh - - name: Configure messaging_address - run: | - sudo /opt/directord/bin/python < /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo cp /usr/local/share/ca-certificates/directord/* /tmp/directord/ - sudo cp /etc/qpid-dispatch/qdrouterd.* /tmp/directord/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-messaging-async-test-logs - path: /tmp/directord* - - functional_zmq_async_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo DRIVER=zeromq bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute async-functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async.yaml \ - --poll \ - --check - - name: Execute functional async-race-condition check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async-race.yaml \ - --poll \ - --check - - name: Execute functional async-time check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 40 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async-time.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-zmq-async-test-logs - path: /tmp/directord* - - functional_defaults_add_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - functional-tests-async-transfer.yaml \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-add-test-logs - path: /tmp/directord* - - functional_bootstrap_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Ensure ssh is installed - run: sudo apt update; sudo apt -y install openssh-server - - name: Check config - run: | - cat > sshd_config < - ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa - - name: Add key to auth file - run: > - cat ~/.ssh/id_rsa.pub | tee -a ~/.ssh/authorized_keys - - name: Ensure the owner of the key is correct - run: | - chmod 600 ~/.ssh/authorized_keys - chmod 700 ~/.ssh - sudo chmod -c 0755 ~/ - ls -la ~/.ssh - - name: Test SSH connection to localhost - run: > - ssh -vvv -i ~/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $(whoami)@localhost - - name: Run bootstrap test - run: | - /opt/directord/bin/directord --debug \ - bootstrap \ - --catalog tools/directord-inventory-catalog.yaml \ - --catalog tests/directord-test-catalog.yaml \ - --key-file ~/.ssh/id_rsa &> /tmp/directord-bootstrap-failure.log - - name: Run bootstrap test for clients only - run: | - /opt/directord/bin/directord --debug \ - bootstrap \ - --catalog tools/directord-inventory-catalog.yaml \ - --catalog tests/directord-test-catalog-no-srvr.yaml \ - --key-file ~/.ssh/id_rsa &> /tmp/directord-bootstrap-failure-no-srvr.log - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-bootstrap-test-logs - path: /tmp/directord* - - functional_defaults_realworld_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Execute functional check - run: | - cd /opt/directord/share/directord/orchestrations - sudo timeout 240 /opt/directord/bin/directord \ - orchestrate \ - {functional-tests-async-race.yaml,functional-tests-async-time.yaml,functional-tests-async.yaml,functional-tests-query.yaml,functional-tests-async-transfer.yaml} \ - --poll \ - --check - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-realworld-test-logs - path: /tmp/directord* - - functional_prune_check: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Run install - run: sudo bash tools/dev-setup.sh /opt/directord python3 ${{ github.workspace }} - - name: Run server service install - run: | - sudo /opt/directord/bin/directord-server-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-server - - name: Run client service install - run: | - sudo /opt/directord/bin/directord-client-systemd - sudo systemctl daemon-reload - sudo systemctl restart directord-client - - name: Wait for client online - run: | - timeout 120 bash -c 'while ! sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Stop client service - run: | - sudo systemctl stop directord-client - - name: Wait for client to be pruned - run: | - timeout 240 bash -c 'while sudo /opt/directord/bin/directord manage --list-nodes; do sleep 1; done' - - name: Generate log details - run: | - sudo journalctl -u directord-client -n 2000 &> /tmp/directord-client.log || true - sudo journalctl -u directord-server -n 2000 &> /tmp/directord-server.log || true - sudo /opt/directord/bin/directord manage --list-nodes &> /tmp/directord-nodes.log || true - sudo /opt/directord/bin/directord manage --dump-cache &> /tmp/directord-cache.log || true - sudo /opt/directord/bin/directord manage --export-jobs /tmp/directord-jobs-export.log || true - sudo cp -R /etc/directord /tmp/ - sudo find /tmp/directord -type d -exec chmod 0755 {} \; - sudo find /tmp/directord -type f -exec chmod 0644 {} \; - if: failure() - - name: Upload build Log artifacts on failure - if: failure() - uses: actions/upload-artifact@v2 - with: - name: functional-prune-logs - path: /tmp/directord* diff --git a/directord/drivers/__init__.py b/directord/drivers/__init__.py index 57f46bf9..6ab3b00b 100644 --- a/directord/drivers/__init__.py +++ b/directord/drivers/__init__.py @@ -305,7 +305,7 @@ def get_machine_id(self): return self.identity def heartbeat_send( - self, host_uptime=None, agent_uptime=None, version=None + self, host_uptime=None, agent_uptime=None, version=None, driver=None ): """Send a heartbeat. @@ -315,6 +315,8 @@ def heartbeat_send( :type agent_uptime: String :param version: Sender directord version :type version: String + :param version: Driver information + :type version: String """ pass diff --git a/directord/interface.py b/directord/interface.py index de34ed38..9f107290 100644 --- a/directord/interface.py +++ b/directord/interface.py @@ -58,21 +58,33 @@ def __init__(self, args): self.keys_exist = os.path.exists( self.public_keys_dir ) and os.path.exists(self.secret_keys_dir) + try: + self.driver = self._load_driver(driver=self.args.driver) + except AttributeError as e: + self.log.warning( + "Falling back with dummy driver due error [ %s ] in driver" + " [ %s ]. Check the driver CLI arguments, the configuration" + " file [ %s ] contents, and ensure all dependencies are" + " installed.", + str(e), + self.args.driver, + self.args.config_file, + ) + self.driver = self._load_driver(driver="dummy") + def _load_driver(self, driver): try: _driver = directord.plugin_import( - plugin=".drivers.{}".format(self.args.driver) + plugin=".drivers.{}".format(driver) ) except Exception as e: raise SystemExit( "Driver was not able to be loaded: {}".format(str(e)) ) else: - self.log.debug( - "Loading messaging driver: [ %s ]", self.args.driver - ) + self.log.debug("Loading messaging driver: [ %s ]", driver) try: - self.driver = _driver.Driver( + return _driver.Driver( args=self.args, encrypted_traffic_data={ "enabled": self.keys_exist, @@ -85,7 +97,7 @@ def __init__(self, args): raise OSError( "Failed to load driver {} - Error: {} - Check" " configuration and dependency installation.".format( - self.args.driver, str(e) + driver, str(e) ) ) from None