V3.0.3 chg digitalocean move (#87)

* CHG digital ocean move

* CHG a bunch of changes for digital ocean

* CHG replace old readme with design doc

* CHG doc name in render.sh

* CHG update design pdf

* CHG update the language to reflect digital ocean on readme

* CHG update design pdf

* CHG fix a few things

* CHG responsive question card feedback

* CHG presort question responses by pool

* FIX some stuff with question editing and assigning

* CHG add zsh shell and optimize xv6 theia image layers

* ADD github org url to course and fix question tests

* CHG fix some things with the anubis cli and management IDE

Makefile

@@ -1,6 +1,6 @@
 PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash
 RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia rpc-regrade
-PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6
+PUSH_SERVICES := api web theia-init theia-proxy theia-admin theia-xv6
 
 
 
@@ -35,7 +35,7 @@ deploy:
 build:
 	docker-compose build --parallel --pull
 
-.PHONY: push         # Push images to registry.osiris.services (requires vpn)
+.PHONY: push         # Push images to registry.digitalocean.com (requires vpn)
 push:
 	docker-compose build --parallel --pull $(PUSH_SERVICES)
 	docker-compose push $(PUSH_SERVICES)
@@ -65,7 +65,7 @@ mindebug:
 	docker-compose up -d traefik db redis-master logstash
 	docker-compose up \
 		-d --force-recreate \
-		api web rpc-default rpc-theia
+		api web-dev rpc-default rpc-theia
 	@echo 'Waiting a moment before running migrations'
 	sleep 3
 	@echo 'running migrations'

api/anubis/models/__init__.py

@@ -7,7 +7,6 @@
 from sqlalchemy_json import MutableJson
 
 from anubis.utils.data import rand
-from anubis.utils.services.logger import logger
 
 db = SQLAlchemy()
 
@@ -90,8 +89,11 @@ class Course(db.Model):
     semester = db.Column(db.TEXT, nullable=True)
     section = db.Column(db.TEXT, nullable=True)
     professor = db.Column(db.TEXT, nullable=False)
-    theia_default_image = db.Column(db.TEXT, nullable=False, default='registry.osiris.services/anubis/xv6')
+    autograde_tests_repo = db.Column(db.TEXT, nullable=False,
+                                     default='https://github.com/os3224/anubis-assignment-tests')
+    theia_default_image = db.Column(db.TEXT, nullable=False, default='registry.digitalocean.com/anubis/xv6')
     theia_default_options = db.Column(MutableJson, default=lambda: {"limits": {"cpu": "2", "memory": "500Mi"}})
+    github_org_url = db.Column(db.TEXT, default='')
 
     @property
     def total_assignments(self):
@@ -177,11 +179,11 @@ class Assignment(db.Model):
     course_id = db.Column(db.String(128), db.ForeignKey(Course.id), index=True)
 
     # Fields
-    name = db.Column(db.TEXT, nullable=False)
+    name = db.Column(db.TEXT, nullable=False, index=True)
     hidden = db.Column(db.Boolean, default=False)
     description = db.Column(db.TEXT, nullable=True)
     github_classroom_url = db.Column(db.TEXT, nullable=True, default=None)
-    pipeline_image = db.Column(db.TEXT, nullable=True)
+    pipeline_image = db.Column(db.TEXT, nullable=True, index=True)
     unique_code = db.Column(
         db.String(8),
         unique=True,
@@ -191,7 +193,7 @@ class Assignment(db.Model):
     accept_late = db.Column(db.Boolean, default=True)
     autograde_enabled = db.Column(db.Boolean, default=True)
     theia_image = db.Column(
-        db.TEXT, default="registry.osiris.services/anubis/theia-xv6"
+        db.TEXT, default="registry.digitalocean.com/anubis/theia-xv6"
     )
     theia_options = db.Column(MutableJson, default=lambda: {})
 
@@ -383,35 +385,27 @@ def data(self):
 
         :return:
         """
-        response = AssignedQuestionResponse.query.filter(
+        from anubis.utils.lms.assignments import get_assignment_due_date
+
+        response: AssignedQuestionResponse = AssignedQuestionResponse.query.filter(
             AssignedQuestionResponse.assigned_question_id == self.id,
         ).order_by(AssignedQuestionResponse.created.desc()).first()
 
-        raw_response = self.question.placeholder
+        response_data = {'submitted': None,'late': True, 'text': self.question.placeholder}
         if response is not None:
-            raw_response = response.response
+            response_data = response.data
 
         return {
             "id": self.id,
-            "response": raw_response,
+            "response": response_data,
             "question": self.question.data,
         }
 
     @property
     def full_data(self):
-        response = AssignedQuestionResponse.query.filter(
-            AssignedQuestionResponse.assigned_question_id == self.id,
-        ).order_by(AssignedQuestionResponse.created.desc()).first()
-
-        raw_response = self.question.placeholder
-        if response is not None:
-            raw_response = response.response
-
-        return {
-            "id": self.id,
-            "question": self.question.full_data,
-            "response": raw_response,
-        }
+        data = self.data
+        data['question'] = self.question.full_data
+        return data
 
 
 class AssignedQuestionResponse(db.Model):
@@ -432,6 +426,16 @@ class AssignedQuestionResponse(db.Model):
     created = db.Column(db.DateTime, default=datetime.now)
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
+    @property
+    def data(self):
+        from anubis.utils.lms.assignments import get_assignment_due_date
+
+        return {
+            'submitted': str(self.created),
+            'late': get_assignment_due_date(self.question.owner, self.question.assignment) < self.created,
+            'text': self.response,
+        }
+
 
 class Submission(db.Model):
     __tablename__ = "submission"
@@ -467,7 +471,7 @@ class Submission(db.Model):
     # Relationships
     owner = db.relationship(User)
     assignment = db.relationship(Assignment)
-    build = db.relationship("SubmissionBuild", cascade="all,delete", backref='submission')
+    build = db.relationship("SubmissionBuild", cascade="all,delete", uselist=False, backref='submission')
     test_results = db.relationship("SubmissionTestResult", cascade="all,delete", backref='submission')
     repo = db.relationship(AssignmentRepo, backref='submissions')
 
@@ -658,7 +662,7 @@ class TheiaSession(db.Model):
     state = db.Column(db.TEXT)
     cluster_address = db.Column(db.TEXT, nullable=True, default=None)
     image = db.Column(
-        db.TEXT, default="registry.osiris.services/anubis/theia-xv6"
+        db.TEXT, default="registry.digitalocean.com/anubis/theia-xv6"
     )
     options = db.Column(MutableJson, nullable=False, default=lambda: dict())
     network_locked = db.Column(db.Boolean, default=True)
@@ -766,6 +770,3 @@ def data(self):
             'assignment_id': self.assignment_id,
             'due_date': str(self.due_date)
         }
-
-
-

api/anubis/rpc/seed.py

@@ -62,6 +62,8 @@ def seed():
     intro_to_os_course = create_course(
         intro_to_os_students,
         name="Intro to OS", course_code="CS-UY 3224", section="A", professor="Gustavo",
+        autograde_tests_repo='https://github.com/os3224/anubis-assignment-tests',
+        github_org_url='https://github.com/os3224',
     )
     os_assignment, _, os_submissions, _ = create_assignment(intro_to_os_course, intro_to_os_students)
     init_submissions(os_submissions)
@@ -75,6 +77,8 @@ def seed():
     mmds_course = create_course(
         mmds_students,
         name="Mining Massive Datasets", course_code="CS-UY 3843", section="A", professor="Gustavo",
+        autograde_tests_repo='https://github.com/os3224/anubis-assignment-tests',
+        github_org_url='https://github.com/os3224'
     )
     mmds_assignment, _, mmds_submissions, _ = create_assignment(mmds_course, mmds_students)
     init_submissions(mmds_submissions)

api/anubis/rpc/theia.py

@@ -52,7 +52,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
     # Init container
     init_container = client.V1Container(
         name="theia-init-{}-{}".format(theia_session.owner.netid, theia_session.id),
-        image="registry.osiris.services/anubis/theia-init:latest",
+        image="registry.digitalocean.com/anubis/theia-init:latest",
         image_pull_policy=os.environ.get("IMAGE_PULL_POLICY", default="Always"),
         env=[
             client.V1EnvVar(name="GIT_REPO", value=theia_session.repo_url),
@@ -91,6 +91,14 @@ def create_theia_pod_obj(theia_session: TheiaSession):
                 name='AUTOSAVE',
                 value='ON' if autosave else 'OFF',
             ),
+            client.V1EnvVar(
+                name='COURSE_ID',
+                value=theia_session.course_id,
+            ),
+            client.V1EnvVar(
+                name='COURSE_CODE',
+                value=theia_session.course.course_code,
+            ),
             *extra_env,
         ],
         resources=client.V1ResourceRequirements(
@@ -111,29 +119,32 @@ def create_theia_pod_obj(theia_session: TheiaSession):
     containers.append(theia_container)
 
     # Sidecar container
-    if autosave:
-        sidecar_container = client.V1Container(
-            name="sidecar",
-            image="registry.osiris.services/anubis/theia-sidecar:latest",
-            image_pull_policy=os.environ.get("IMAGE_PULL_POLICY", default="Always"),
-            env=[
-                client.V1EnvVar(
-                    name="GIT_CRED",
-                    value_from=client.V1EnvVarSource(
-                        secret_key_ref=client.V1SecretKeySelector(
-                            name="git", key="credentials"
-                        )
-                    ),
+    sidecar_container = client.V1Container(
+        name="sidecar",
+        image="registry.digitalocean.com/anubis/theia-sidecar:latest",
+        image_pull_policy=os.environ.get("IMAGE_PULL_POLICY", default="Always"),
+        env=[
+            client.V1EnvVar(
+                name="GIT_CRED",
+                value_from=client.V1EnvVarSource(
+                    secret_key_ref=client.V1SecretKeySelector(
+                        name="git", key="credentials"
+                    )
                 ),
-            ],
-            volume_mounts=[
-                client.V1VolumeMount(
-                    mount_path="/home/project",
-                    name=volume_name,
-                )
-            ],
-        )
-        containers.append(sidecar_container)
+            ),
+            client.V1EnvVar(
+                name='AUTOSAVE',
+                value='ON' if autosave else 'OFF',
+            ),
+        ],
+        volume_mounts=[
+            client.V1VolumeMount(
+                mount_path="/home/project",
+                name=volume_name,
+            )
+        ],
+    )
+    containers.append(sidecar_container)
 
     extra_labels = {}
     spec_extra = {}
@@ -238,13 +249,17 @@ def initialize_theia_session(theia_session_id: str):
     name = get_theia_pod_name(theia_session)
     n = 10
     while True:
+
+        # Get the pod information from the kubernetes api
         pod: client.V1Pod = v1.read_namespaced_pod(
             name=name,
             namespace="anubis",
         )
 
         if pod.status.phase == "Pending":
             n += 1
+
+            # Wait at 60 iterations while it is pending before giving up
             if n > 60:
                 logger.error(
                     "Theia session took too long to initialize. Freeing worker."
@@ -254,8 +269,17 @@ def initialize_theia_session(theia_session_id: str):
             time.sleep(1)
 
         if pod.status.phase == "Running":
+            # Set the cluster address and state
             theia_session.cluster_address = pod.status.pod_ip
             theia_session.state = "Running"
+
+            # We need to introduce a small amount of time here
+            # to give the theia server a moment to actually run
+            # before the button on the web frontend to go to
+            # the session is available to the user
+            time.sleep(1)
+
+            # Index the event
             esindex(
                 "theia",
                 body={
@@ -264,6 +288,8 @@ def initialize_theia_session(theia_session_id: str):
                     "netid": theia_session.owner.netid,
                 },
             )
+
+            # Log the event
             logger.info("Theia session started {}".format(name))
             break
 

api/anubis/utils/auth.py

@@ -5,8 +5,7 @@
 from typing import Union
 
 import jwt
-from flask import g
-from flask import request
+from flask import g, request, has_request_context
 
 from anubis.config import config
 from anubis.models import User, TAForCourse, ProfessorForCourse
@@ -74,6 +73,9 @@ def get_token() -> Union[str, None]:
     :return:
     """
 
+    if not has_request_context():
+        return None
+
     return request.headers.get("token", default=None) or request.cookies.get(
         "token", default=None
     ) or request.args.get('token', default=None)

api/anubis/utils/data.py

@@ -326,8 +326,8 @@ def wrapper(*args, **kwargs):
 
         # Push an app context
         with app.app_context():
-
-            # Call the function within an app context
-            return function(*args, **kwargs)
+            with app.test_request_context():
+                # Call the function within an app context
+                return function(*args, **kwargs)
 
     return wrapper

api/anubis/utils/lms/assignments.py

@@ -153,7 +153,7 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
         )
     ).first()
     if course is None:
-        return "Unable to find class", False
+        return "Unable to find course", False
 
     assert_course_admin(course.id)