gh actions gleanerio#6

internetofwater · May 29, 2024 · 9d972e0 · 9d972e0
1 parent 7dbea76
commit 9d972e0
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 3 deletions.
diff --git a/.github/workflows/test_geoconnex_dev_sitemap.yml b/.github/workflows/test_geoconnex_dev_sitemap.yml
@@ -14,9 +14,9 @@ jobs:
       - run: sudo apt-get install git 
       - run: mkdir -p $HOME/bin
       - run: curl https://dl.min.io/client/mc/release/linux-amd64/mc -o $HOME/bin/mc && chmod +x $HOME/bin/mc 
-      - run: scripts/iow/start-headless-chrome.sh
-      - run: scripts/iow/start-minio.sh && sleep 3
+      - run: sh scripts/iow/start-headless-chrome.sh
+      - run: sh scripts/iow/start-minio.sh && sleep 3
       - run: $HOME/bin/mc alias set myminio http://localhost:9000 amazingaccesskey amazingsecretkey
       - run: mc mb myminio/iow
       - run: make 
-      - run: scripts/iow/start-gleaner.sh      
+      - run: sh scripts/iow/start-gleaner.sh      
diff --git a/scripts/iow/start-gleaner.sh b/scripts/iow/start-gleaner.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+
+
+
+TS=`date +%Y-%m-%dT%H.%M.%S`
+LOGDIR="$HOME/logs/$TS"
+mkdir -p $LOGDIR || exit 1
+cd $LOGDIR || exit 1
+
+for src in `cat ~/conf/gleanerconfig.yaml | grep '\Wname:'|awk '{print $2}'`
+do
+
+OUTFILE="$LOGDIR/gleaner-$src.out"
+ERRFILE="$LOGDIR/gleaner-$src.err"
+
+echo "harvesting source '$src'..."
+#strace -f -o $LOGDIR/strace-$src.out gleaner -cfg $HOME/conf/gleanerconfig.yaml -source $src -rude > $OUTFILE 2>$ERRFILE
+gleaner -log debug  -cfg $HOME/conf/gleanerconfig.yaml -source $src -rude > $OUTFILE 2>$ERRFILE
+done
+echo "complete!"
+
diff --git a/scripts/iow/start-headless-chrome.sh b/scripts/iow/start-headless-chrome.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+
+docker run -d  \
+	--rm \
+	-p 9222:9222 \
+	--name hc \
+	chromedp/headless-shell
+
diff --git a/scripts/iow/start-minio.sh b/scripts/iow/start-minio.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# --rm
+# --memory
+docker run -d \
+   --rm \
+   -p 9000:9000 \
+   -p 9001:9001 \
+   --name minio \
+   -e "MINIO_ROOT_USER=amazingaccesskey" \
+   -e "MINIO_ROOT_PASSWORD=amazingsecretkey" \
+   quay.io/minio/minio server /data --console-address ":9001"
+
+
diff --git a/scripts/iow/stop-headless-chrome.sh b/scripts/iow/stop-headless-chrome.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+
+docker stop hc
+docker rm hc > /dev/null 2>&1
+
+
diff --git a/scripts/iow/stop-minio.sh b/scripts/iow/stop-minio.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+
+docker stop minio
+docker rm minio >/dev/null 2>&1
+