CHG extend admin panel (#82)

* ADD hidden tests * CHG minor admin panel UI improvements * ADD theia post * CHG update theia post * FIX datagrid height stuff * CHG minor import fixes * FIX off by one in data-grid * FIX off by one in static data-grid * CHG reorange the elevator pitch * CHG rename kube to k8s * CHG move helm chart to seperate directory * RM scripts * CHG move values file into chart * ADD queries * ADD hidden icon to admin panel tests * CHG detect and skip state updates for hidden tests * RM demo * RM cli symlink * CHG improve networkpolicy * CHG make api static files filename aware * CHG updates to blog format * CHG let admins start ides before release * CHG refactor utils * CHG update redis helm chart to the 2.0 version of redis chart * ADD first very much work in progress visual * ADD radial solve count with selection tabs * CHG consistent usage of app.kubernetes.io/name in labels * ADD light LRU caching to theia-proxy service * CHG consistent usage of app.kubernetes.io/name in rpc * CHG consistent font color in nav bar and hide auth chip when anon * CHG move data.py and decorators.py * CHG make visual sql queries course aware * CHG fix weird spacing in index.html * CHG make copyright open in new tab * FIX minor typographical mistakes in midterm post * FIX import in visual job * ADD WIP react-vis usage graph * CHG comment out api network policy
AnubisLMS · Apr 28, 2021 · 1c2e1d1 · 1c2e1d1
1 parent 1fbd63a
commit 1c2e1d1
Show file tree

Hide file tree

Showing 144 changed files with 2,372 additions and 1,770 deletions.
diff --git a/Makefile b/Makefile
@@ -33,8 +33,8 @@ help:
 
 .PHONY: deploy       # Deploy Anubis to cluster
 deploy:
-	./kube/deploy.sh
-	./kube/restart.sh
+	./k8s/deploy.sh
+	./k8s/restart.sh
 
 .PHONY: build        # Build all docker images
 build:
@@ -77,15 +77,15 @@ mindebug:
 
 .PHONY: mkdebug      # Start minikube debug
 mkdebug:
-	./kube/debug/provision.sh
+	./k8s/debug/provision.sh
 	@echo ''
 	@echo 'seed: http://localhost/api/admin/seed/'
 	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
 	@echo 'site: http://localhost/'
 
 .PHONY: mkrestart    # Restart minikube debug
 mkrestart:
-	./kube/debug/restart.sh
+	./k8s/debug/restart.sh
 	@echo ''
 	@echo 'seed: http://localhost/api/admin/seed/'
 	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'

diff --git a/api/anubis/app.py b/api/anubis/app.py
@@ -10,9 +10,9 @@ def init_services(app):
     """
     from anubis.config import config
     from anubis.models import db, Config
-    from anubis.utils.cache import cache, cache_health
-    from anubis.utils.migrate import migrate
-    from anubis.utils.elastic import add_global_error_handler
+    from anubis.utils.services.cache import cache, cache_health
+    from anubis.utils.services.migrate import migrate
+    from anubis.utils.services.elastic import add_global_error_handler
 
     # Init services
     db.init_app(app)

diff --git a/api/anubis/config.py b/api/anubis/config.py
@@ -64,7 +64,7 @@ def __init__(self):
         self.THEIA_DOMAIN = os.environ.get(
             "THEIA_DOMAIN", default="ide.anubis.osiris.services"
         )
-        self.THEIA_TIMEOUT = timedelta(hours=6) if not self.DEBUG else timedelta(minutes=2)
+        self.THEIA_TIMEOUT = timedelta(hours=6)
 
         logging.info(
             "Starting with DATABASE_URI: {}".format(self.SQLALCHEMY_DATABASE_URI)

diff --git a/api/anubis/models/__init__.py b/api/anubis/models/__init__.py
@@ -7,7 +7,7 @@
 from sqlalchemy_json import MutableJson
 
 from anubis.utils.data import rand
-from anubis.utils.logger import logger
+from anubis.utils.services.logger import logger
 
 db = SQLAlchemy()
 
@@ -165,9 +165,15 @@ def data(self):
             "ide_enabled": self.ide_enabled,
             "ide_active": self.due_date + timedelta(days=3 * 7) > datetime.now(),
             "github_classroom_link": self.github_classroom_url,
-            "tests": [t.data for t in self.tests],
+            "tests": [t.data for t in self.tests if t.hidden is False],
         }
 
+    @property
+    def full_data(self):
+        data = self.data
+        data['tests'] = [t.data for t in self.tests]
+        return data
+
     @property
     def meta_shape(self):
         return {
@@ -228,13 +234,18 @@ class AssignmentTest(db.Model):
 
     # Fields
     name = db.Column(db.String(128), index=True)
+    hidden = db.Column(db.Boolean, default=False)
 
     # Relationships
     assignment = db.relationship(Assignment)
 
     @property
     def data(self):
-        return {"name": self.name}
+        return {
+            "id": self.id,
+            "name": self.name,
+            "hidden": self.hidden
+        }
 
 
 class AssignmentQuestion(db.Model):
@@ -440,18 +451,14 @@ def init_submission_models(self):
         # Commit new models
         db.session.commit()
 
-    @property
-    def url(self):
-        return "https://anubis.osiris.services/submission/{}".format(self.commit)
-
     @property
     def netid(self):
         if self.owner is not None:
             return self.owner.netid
         return "null"
 
     @property
-    def tests(self):
+    def visible_tests(self):
         """
         Get a list of dictionaries of the matching Test, and TestResult
         for the current submission.
@@ -460,11 +467,30 @@ def tests(self):
         """
 
         # Query for matching AssignmentTests, and TestResults
-        tests = SubmissionTestResult.query.filter_by(
-            submission_id=self.id,
+        tests = SubmissionTestResult.query.join(AssignmentTest).filter(
+            SubmissionTestResult.submission_id == self.id,
+            AssignmentTest.hidden == False,
         ).all()
 
-        logger.debug("Loaded tests {}".format(tests))
+        # Convert to dictionary data
+        return [
+            {"test": result.assignment_test.data, "result": result.data}
+            for result in tests
+        ]
+
+    @property
+    def all_tests(self):
+        """
+        Get a list of dictionaries of the matching Test, and TestResult
+        for the current submission.
+
+        :return:
+        """
+
+        # Query for matching AssignmentTests, and TestResults
+        tests = SubmissionTestResult.query.join(AssignmentTest).filter(
+            SubmissionTestResult.submission_id == self.id,
+        ).all()
 
         # Convert to dictionary data
         return [
@@ -493,7 +519,18 @@ def full_data(self):
 
         # Add connected models
         data["repo"] = self.repo.repo_url
-        data["tests"] = self.tests
+        data["tests"] = self.visible_tests
+        data["build"] = self.build.data if self.build is not None else None
+
+        return data
+
+    @property
+    def admin_data(self):
+        data = self.data
+
+        # Add connected models
+        data["repo"] = self.repo.repo_url
+        data["tests"] = self.all_tests
         data["build"] = self.build.data if self.build is not None else None
 
         return data
@@ -622,7 +659,7 @@ class TheiaSession(db.Model):
 
     @property
     def data(self):
-        from anubis.utils.theia import theia_redirect_url
+        from anubis.utils.services.theia import theia_redirect_url
 
         return {
             "id": self.id,

diff --git a/api/anubis/rpc/batch.py b/api/anubis/rpc/batch.py
@@ -1,9 +1,9 @@
-from anubis.utils.logger import logger
+from anubis.utils.services.logger import logger
 
 
 def rpc_bulk_regrade(submissions):
     from anubis.app import create_app
-    from anubis.utils.submissions import bulk_regrade_submission
+    from anubis.utils.assignment.submissions import bulk_regrade_submission
 
     app = create_app()
 

diff --git a/api/anubis/rpc/pipeline.py b/api/anubis/rpc/pipeline.py
@@ -7,7 +7,7 @@
 
 from anubis.models import Config, Submission
 from anubis.utils.data import is_debug
-from anubis.utils.logger import logger
+from anubis.utils.services.logger import logger
 
 
 def create_pipeline_job_obj(client, submission):
@@ -51,7 +51,7 @@ def create_pipeline_job_obj(client, submission):
     # Create and configure a spec section
     template = client.V1PodTemplateSpec(
         metadata=client.V1ObjectMeta(
-            labels={"app": "submission-pipeline", "role": "submission-pipeline-worker"}
+            labels={"app.kubernetes.io/name": "submission-pipeline", "role": "submission-pipeline-worker"}
         ),
         spec=client.V1PodSpec(restart_policy="Never", containers=[container]),
     )
@@ -108,7 +108,7 @@ def create_submission_pipeline(submission_id: str):
     :param submission_id: submission.id of to test
     """
     from anubis.app import create_app
-    from anubis.utils.rpc import enqueue_autograde_pipeline
+    from anubis.utils.services.rpc import enqueue_autograde_pipeline
 
     app = create_app()
 

diff --git a/api/anubis/rpc/seed.py b/api/anubis/rpc/seed.py
@@ -19,7 +19,7 @@
     AssignedQuestionResponse,
 )
 from anubis.utils.data import rand
-from anubis.utils.questions import assign_questions
+from anubis.utils.assignment.questions import assign_questions
 
 
 def rand_str(n=10):

diff --git a/api/anubis/rpc/theia.py b/api/anubis/rpc/theia.py
@@ -5,9 +5,9 @@
 from kubernetes import config, client
 
 from anubis.models import db, Config, TheiaSession
-from anubis.utils.elastic import esindex
-from anubis.utils.logger import logger
-from anubis.utils.auth import create_token
+from anubis.utils.services.elastic import esindex
+from anubis.utils.services.logger import logger
+from anubis.utils.users.auth import create_token
 import base64
 
 
@@ -25,7 +25,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         metadata=client.V1ObjectMeta(
             name=volume_name,
             labels={
-                "app": "theia",
+                "app.kubernetes.io/name": "theia",
                 "role": "session-storage",
                 "netid": theia_session.owner.netid,
                 "session": theia_session.id,
@@ -66,20 +66,12 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         ],
     )
 
-    limits = {"cpu": "2", "memory": "500Mi"}
-    if "limits" in theia_session.options:
-        limits = theia_session.options["limits"]
-
-    requests = {"cpu": "250m", "memory": "100Mi"}
-    if "requests" in theia_session.options:
-        requests = theia_session.options["requests"]
-
-    autosave = True
-    if 'autosave' in theia_session.options:
-        autosave = theia_session.options['autosave']
+    limits = theia_session.options.get('limits', {"cpu": "2", "memory": "500Mi"})
+    requests = theia_session.options.get('requests', {"cpu": "250m", "memory": "100Mi"})
+    autosave = theia_session.options.get('autosave', True)
 
     extra_env = []
-    if theia_session.owner.is_admin or theia_session.owner.is_superuser:
+    if theia_session.options.get('credentials', False):
         extra_env.append(client.V1EnvVar(
             name='INCLUSTER',
             value=base64.b64encode(create_token(theia_session.owner.netid).encode()).decode(),
@@ -110,6 +102,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         ],
         security_context=client.V1SecurityContext(
             privileged=theia_session.privileged,
+            # capabilities=client.V1Capabilities(add=['SYS_ADMIN', 'NET_ADMIN'])
         ),
     )
     containers.append(theia_container)
@@ -141,11 +134,19 @@ def create_theia_pod_obj(theia_session: TheiaSession):
 
     extra_labels = {}
     spec_extra = {}
+
+    # If network locked, then set the network policy to student
+    # and dns to 1.1.1.1
     if theia_session.network_locked:
         extra_labels["network-policy"] = "student"
         spec_extra['dns_policy'] = "None"
         spec_extra["dns_config"] = client.V1PodDNSConfig(nameservers=["1.1.1.1"])
 
+    # If the network is not locked, then we still need to apply
+    # the admin policy.
+    else:
+        extra_labels["network-policy"] = "admin"
+
     # Create pod
     pod = client.V1Pod(
         spec=client.V1PodSpec(
@@ -158,7 +159,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         metadata=client.V1ObjectMeta(
             name="theia-{}-{}".format(theia_session.owner.netid, theia_session.id),
             labels={
-                "app": "theia",
+                "app.kubernetes.io/name": "theia",
                 "role": "theia-session",
                 "netid": theia_session.owner.netid,
                 "session": theia_session.id,
@@ -205,7 +206,7 @@ def initialize_theia_session(theia_session_id: str):
                     theia_session_id
                 )
             )
-            from anubis.utils.rpc import enqueue_ide_initialize
+            from anubis.utils.services.rpc import enqueue_ide_initialize
 
             enqueue_ide_initialize(theia_session_id)
             return
@@ -290,7 +291,7 @@ def reap_theia_session_resources(theia_session_id: str):
     # Delete the pods
     v1.delete_collection_namespaced_pod(
         namespace="anubis",
-        label_selector="app=theia,role=theia-session,session={}".format(
+        label_selector="app.kubernetes.io/name=theia,role=theia-session,session={}".format(
             theia_session_id
         ),
         propagation_policy="Background",
@@ -299,7 +300,7 @@ def reap_theia_session_resources(theia_session_id: str):
     # Delete the pvc
     v1.delete_collection_namespaced_persistent_volume_claim(
         namespace="anubis",
-        label_selector="app=theia,role=session-storage,session={}".format(
+        label_selector="app.kubernetes.io/name=theia,role=session-storage,session={}".format(
             theia_session_id
         ),
         propagation_policy="Background",
@@ -311,7 +312,7 @@ def list_theia_pods():
     v1 = client.CoreV1Api()
 
     pods = v1.list_namespaced_pod(
-        namespace="anubis", label_selector="app=theia,role=theia-session"
+        namespace="anubis", label_selector="app.kubernetes.io/name=theia,role=theia-session"
     )
 
     return pods
@@ -459,7 +460,7 @@ def reap_stale_theia_sessions(*_):
 
     with app.app_context():
         resp = v1.list_namespaced_pod(
-            namespace="anubis", label_selector="app=theia,role=theia-session"
+            namespace="anubis", label_selector="app.kubernetes.io/name=theia,role=theia-session"
         )
 
         for n, pod in enumerate(resp.items):

diff --git a/api/anubis/rpc/visualizations.py b/api/anubis/rpc/visualizations.py
@@ -1,4 +1,4 @@
-from anubis.utils.visualizations import get_usage_plot
+from anubis.utils.visuals.usage import get_usage_plot
 
 
 def create_visuals(*_, **__):

diff --git a/api/anubis/utils/assignment/__init__.py b/api/anubis/utils/assignment/__init__.py
diff --git a/api/anubis/utils/assignments.py → api/anubis/utils/assignment/assignments.py b/api/anubis/utils/assignments.py → api/anubis/utils/assignment/assignments.py
@@ -16,11 +16,11 @@
     AssignmentRepo,
     SubmissionTestResult,
 )
-from anubis.utils.auth import get_user
-from anubis.utils.cache import cache
+from anubis.utils.users.auth import get_user
+from anubis.utils.services.cache import cache
 from anubis.utils.data import is_debug
-from anubis.utils.logger import logger
-from anubis.utils.questions import ingest_questions
+from anubis.utils.services.logger import logger
+from anubis.utils.assignment.questions import ingest_questions
 
 
 @cache.memoize(timeout=5, unless=is_debug)