Merge pull request #10 from stackql/feature/updates

Feature/updates

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Description
+
+Please include a summary of the changes and the related issue(s). Please also include relevant motivation and context.
+
+Fixes #
+
+## Checklist
+
+Please make sure that the following criteria are met:
+
+- [ ] The PR title is descriptive.
+- [ ] Include documentation for example queries (comments or markdown in notebooks)
+- [ ] I have ⭐'ed the [stackql](https://github.com/stackql/stackql) and [stackql-jupyter-demo](https://github.com/stackql/stackql-jupyter-demo) repos.
+
+## Additional Notes
+
+Add any additional information or context that might help the reviewers.

.github/workflows/main.yml

@@ -3,7 +3,6 @@ name: Build and Publish
 on:
   push:
     branches: [main]
-  pull_request:
 
 jobs:
   build-and-push-docker-image:

.github/workflows/star-check.yml

@@ -0,0 +1,45 @@
+name: Check if PR author has starred the repository
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  check-starred:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4.1.7
+
+    - name: Get PR author username
+      id: get-author
+      run: echo "username=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV
+
+    - name: Pull github provider
+      uses: stackql/stackql-exec@v2.2.1
+      with:
+        is_command: 'true'
+        query: "REGISTRY PULL github;"
+
+    - name: Run stackql query
+      id: check-star
+      uses: stackql/stackql-assert@v2.2.1
+      with:
+        test_query: |
+          SELECT repo, count(*) as has_starred
+          FROM github.activity.repo_stargazers
+          WHERE owner = 'stackql' and repo in ('stackql', 'stackql-jupyter-demo') and login = '${{ env.username }}'
+          GROUP BY repo;
+        expected_results_str: '[{"has_starred":"1","repo":"stackql"},{"has_starred":"1","repo":"stackql-jupyter-demo"}]'
+      continue-on-error: true
+
+    - name: Check if starred
+      if: always() # Ensures this runs regardless of check-star outcome
+      run: |
+        if [ "${{ steps.check-star.outcome }}" = "success" ]; then
+          echo "::notice::Thanks for your support!"
+        else
+          echo "::error::It seems you haven't starred the StackQL repositories. Please star the following repos before proceeding: https://github.com/stackql/stackql-jupyter-demo (this repo) and https://github.com/stackql/stackql (our core repo)"
+          exit 1
+        fi
+        

Dockerfile

@@ -1,3 +1,71 @@
+# FROM stackql/stackql:latest AS stackql
+# EXPOSE 5444
+# WORKDIR /home/stackql
+# RUN adduser --system --uid 1001 stackql
+# RUN addgroup --system --gid 1001 stackql
+# RUN chown stackql:stackql /home/stackql
+# RUN chown stackql:stackql /srv
+# USER stackql
+# # pull stackql providers
+# RUN stackql exec 'registry pull aws' || (echo "Failed to pull aws provider" && exit 1)
+# RUN stackql exec 'registry pull google' || (echo "Failed to pull google provider" && exit 1)
+# RUN stackql exec 'registry pull github' || (echo "Failed to pull github provider" && exit 1)
+# RUN stackql exec 'registry pull azure' || (echo "Failed to pull azure provider" && exit 1)
+# # RUN stackql exec 'registry pull k8s'
+# # RUN stackql exec 'registry pull netlify'
+# # RUN stackql exec 'registry pull okta'
+# # RUN stackql exec 'registry pull sumologic'
+# # RUN stackql exec 'registry pull digitalocean'
+
+# FROM jupyter/base-notebook:latest AS jupyter
+# WORKDIR /jupyter
+# USER root
+# RUN apt-get update && \
+#     apt-get upgrade -y
+# # copy example notebooks to Jupyter workspace
+# COPY ./notebooks/ /jupyter/
+# RUN chmod 644 *.ipynb && \
+#     chown jovyan:users *.ipynb
+# # copy magic extensions
+# RUN mkdir -p /jupyter/ext
+# COPY ./extensions/* /jupyter/ext
+# RUN chmod 644 /jupyter/ext/*.py && \
+#     chown jovyan:users /jupyter/ext/*.py
+# # copy entrypoint script
+# RUN mkdir -p /scripts
+# COPY ./scripts/start-server.sh /scripts
+# COPY ./scripts/entrypoint.sh /scripts
+# RUN chmod +x /scripts/start-server.sh
+# RUN chmod +x /scripts/entrypoint.sh
+# # set up matplotlib temp dir
+# RUN mkdir -p /tmp/matplotlib
+# RUN chmod 777 /tmp/matplotlib
+# ENV MPLCONFIGDIR=/tmp/matplotlib
+# ENV PYDEVD_DISABLE_FILE_VALIDATION=1
+# # setup python environment
+# ENV PYTHON_PACKAGES="\
+#     pystackql>=3.6.4 \
+#     matplotlib \
+#     pandas \
+# 	mplfinance \
+#     psycopg2-binary \
+#     nest_asyncio \
+#     plotly \
+#     ipytree \
+#     nbformat \
+#     networkx \
+# " 
+# RUN pip install --upgrade pip \
+#     && pip install --no-cache-dir $PYTHON_PACKAGES
+# # copy stackql providers from stackql container 
+# COPY --from=stackql /home/stackql/.stackql /jupyter/.stackql
+# RUN ls -al /jupyter/.stackql/src/aws || (echo "aws provider not present" && exit 1)
+# RUN ls -al /jupyter/.stackql/src/googleapis.com || (echo "google provider not present" && exit 1)
+# RUN ls -al /jupyter/.stackql/src/github || (echo "github provider not present" && exit 1)
+# # copy stackql binary from stackql container (service instance)
+# COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql
+
+# Stage 1: StackQL setup
 FROM stackql/stackql:latest AS stackql
 EXPOSE 5444
 WORKDIR /home/stackql
@@ -6,59 +74,70 @@ RUN addgroup --system --gid 1001 stackql
 RUN chown stackql:stackql /home/stackql
 RUN chown stackql:stackql /srv
 USER stackql
-# pull stackql providers
+
+# Pull stackql providers
 RUN stackql exec 'registry pull aws' || (echo "Failed to pull aws provider" && exit 1)
 RUN stackql exec 'registry pull google' || (echo "Failed to pull google provider" && exit 1)
 RUN stackql exec 'registry pull github' || (echo "Failed to pull github provider" && exit 1)
 RUN stackql exec 'registry pull azure' || (echo "Failed to pull azure provider" && exit 1)
-# RUN stackql exec 'registry pull k8s'
-# RUN stackql exec 'registry pull netlify'
-# RUN stackql exec 'registry pull okta'
-# RUN stackql exec 'registry pull sumologic'
-# RUN stackql exec 'registry pull digitalocean'
 
+# Stage 2: Jupyter setup
 FROM jupyter/base-notebook:latest AS jupyter
 WORKDIR /jupyter
 USER root
 RUN apt-get update && \
     apt-get upgrade -y
-# copy example notebooks to Jupyter workspace
+
+# Copy example notebooks to Jupyter workspace
 COPY ./notebooks/ /jupyter/
-RUN chmod 644 *.ipynb && \
-    chown jovyan:users *.ipynb
-# copy magic extensions
+RUN chmod 644 /jupyter/*.ipynb && \
+    chown jovyan:users /jupyter/*.ipynb
+
+# Copy magic extensions
 RUN mkdir -p /jupyter/ext
-COPY ./extensions/* /jupyter/ext
+COPY ./extensions/* /jupyter/ext/
 RUN chmod 644 /jupyter/ext/*.py && \
     chown jovyan:users /jupyter/ext/*.py
-# copy entrypoint script
+
+# Copy entrypoint script
 RUN mkdir -p /scripts
+COPY ./scripts/start-server.sh /scripts
 COPY ./scripts/entrypoint.sh /scripts
-RUN chmod +x /scripts/entrypoint.sh
-# set up matplotlib temp dir
+RUN chmod +x /scripts/start-server.sh /scripts/entrypoint.sh
+
+# Set up matplotlib temp dir
 RUN mkdir -p /tmp/matplotlib
 RUN chmod 777 /tmp/matplotlib
 ENV MPLCONFIGDIR=/tmp/matplotlib
 ENV PYDEVD_DISABLE_FILE_VALIDATION=1
-# setup python environment
+
+# Setup python environment
 ENV PYTHON_PACKAGES="\
     pystackql>=3.6.4 \
     matplotlib \
     pandas \
-	mplfinance \
+    mplfinance \
     psycopg2-binary \
     nest_asyncio \
     plotly \
     ipytree \
     nbformat \
     networkx \
 " 
-RUN pip install --upgrade pip \
-    && pip install --no-cache-dir $PYTHON_PACKAGES
-# copy stackql providers from stackql container 
+RUN pip install --upgrade pip && pip install --no-cache-dir $PYTHON_PACKAGES
+
+# Copy stackql providers and binary from stackql container
 COPY --from=stackql /home/stackql/.stackql /jupyter/.stackql
+COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql
+
+# Ensure providers are present
 RUN ls -al /jupyter/.stackql/src/aws || (echo "aws provider not present" && exit 1)
 RUN ls -al /jupyter/.stackql/src/googleapis.com || (echo "google provider not present" && exit 1)
 RUN ls -al /jupyter/.stackql/src/github || (echo "github provider not present" && exit 1)
-# copy stackql binary from stackql container (service instance)
-COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql
+
+# Set user back to default jovyan
+USER jovyan
+
+# Set entrypoint
+# ENTRYPOINT ["/scripts/entrypoint.sh"]
+# CMD ["start-notebook.sh"]

README.md

@@ -1,3 +1,14 @@
+> ⚡ **Calling All Cloud/Data/Security Enthusiasts, Hacktoberfest 2024 is here!** ⚡  
+> Interested in contributing StackQL (SQL) queries, notebooks and visualizations for **Cloud Security Posture Management (CSPM)**, **FinOps**, **Cloud Inventory Analysis**, or **Infrastructure-as-Code (IaC)**?  
+>
+> Check out the issues and get started with your first pull request!, Let’s build something amazing together this Hacktoberfest!  
+
+💡 **Explore our repositories:** [StackQL](https://github.com/stackql/stackql), [StackQL Deploy](https://stackql-deploy.io/docs/), find provider documentation in the [StackQL Provider Registry Docs](https://registry.stackql.io/)  
+
+🔎 Build out example queries for [`aws`](https://aws.stackql.io/providers/aws/), [`gcp`](https://google.stackql.io/providers/google/), [`azure`](https://azure.stackql.io/providers/azure/), [`digitalocean`](https://digitalocean.stackql.io/providers/digitalocean/), [`linode`](https://linode.stackql.io/providers/linode/), [`okta`](https://okta.stackql.io/providers/okta/) and more, including multicloud queries!  
+
+---
+
 [![StackQL](https://stackql.io/img/stackql-logo-bold.png)](https://stackql.io/)  
 <br />
 [![GitHub Actions](https://github.com/stackql/stackql-jupyter-demo/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/stackql/stackql-jupyter-demo/actions/workflows/main.yml)

scripts/entrypoint.sh

@@ -1,6 +1,2 @@
 #!/bin/sh
-/srv/stackql/stackql --version
-echo "starting stackql server..."
-nohup /srv/stackql/stackql --execution.concurrency.limit=-1 --dataflow.components.max=50 --dataflow.dependency.max=50 --http.response.pageLimit=-1 --pgsrv.port=5466 srv &
-echo "stackql server started"
-start-notebook.sh --NotebookApp.token=''
+sh /scripts/start-server.sh && start-notebook.sh --NotebookApp.token=''

scripts/start-server.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+/srv/stackql/stackql --version
+echo "starting stackql server..."
+nohup /srv/stackql/stackql --execution.concurrency.limit=-1 --dataflow.components.max=50 --dataflow.dependency.max=50 --http.response.pageLimit=-1 --pgsrv.port=5466 srv &
+echo "stackql server started"