V2.2.8 fix midterm spring2021 (#80)

* ADD more detail to admin theia page

* ADD usage visuals

* FIX visual labels

* FIX visual jsx format

* FIX visuals context

* FIX pipeline api stability

* ADD assignment/submission blog post

* CHG reorganize rpc into seperate queues for faster ide start during regrading

* ADD packaging post

* CHG handle svg image content type correctly

* ADD sequential response tracking

* CHG split rpc queues in compose debug

* FIX versioned question response on full_data prop

* CHG completely rework theia-admin docker build

* ADD incluster anubis cli authentication

Makefile

@@ -1,5 +1,5 @@
 PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer
-RESTART_ALWAYS_SERVICES := api web-dev rpc-worker
+RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia
 PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6
 
 

api/Dockerfile

@@ -1,17 +1,19 @@
-FROM python:3.9-alpine
+FROM python:3.9-slim-buster
 
 ENV SECRET_KEY=DEFAULT
 
 WORKDIR /opt/app
 
 COPY requirements.txt requirements.txt
 
-RUN apk add --update --no-cache mariadb-client git tzdata gcc g++ musl-dev libmagic \
+RUN apt update \
+  && apt install -y mariadb-client git tzdata gcc g++ libmagic-dev \
   && pip3 install --no-cache-dir -r ./requirements.txt \
-  && adduser -D anubis \
+  && useradd -M anubis \
   && chown anubis:anubis -R /opt/app \
   && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
-  && apk del gcc g++ \
+  && apt purge -y gcc g++ \
+  && apt autoremove -y \
   && env USER=root find /opt/app -depth \
   \( \
   \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
@@ -22,8 +24,12 @@ RUN apk add --update --no-cache mariadb-client git tzdata gcc g++ musl-dev libma
   \( \
   \( -type d -a \( -name test -o -name tests -o -name idle_test \) \) \
   -o \( -type f -a \( -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) \) \
-  \) -exec rm -rf '{}' +
-
+  \) -exec rm -rf '{}' + \
+  && rm -rf /usr/share/doc \
+  && rm -rf /usr/lib/gcc \
+  && rm -rf /usr/local/share/.cache \
+  && rm -rf /var/cache/apt/* \
+  && rm -rf /var/lib/apt/lists/*
 
 USER anubis
 

api/anubis/models/__init__.py

@@ -115,8 +115,8 @@ class InCourse(db.Model):
     owner_id = db.Column(db.String(128), db.ForeignKey(User.id), primary_key=True)
     course_id = db.Column(db.String(128), db.ForeignKey(Course.id), primary_key=True)
 
-    owner = db.relationship(User, cascade="all,delete")
-    course = db.relationship(Course, cascade="all,delete")
+    owner = db.relationship(User)
+    course = db.relationship(Course)
 
 
 class Assignment(db.Model):
@@ -149,7 +149,7 @@ class Assignment(db.Model):
     due_date = db.Column(db.DateTime, nullable=False)
     grace_date = db.Column(db.DateTime, nullable=True)
 
-    course = db.relationship(Course, cascade="all,delete", backref="assignments")
+    course = db.relationship(Course, backref="assignments")
     tests = db.relationship("AssignmentTest", cascade="all,delete")
     repos = db.relationship("AssignmentRepo", cascade="all,delete")
 
@@ -203,8 +203,8 @@ class AssignmentRepo(db.Model):
     repo_url = db.Column(db.String(128), nullable=False)
 
     # Relationships
-    owner = db.relationship(User, cascade="all,delete")
-    assignment = db.relationship(Assignment, cascade="all,delete")
+    owner = db.relationship(User)
+    assignment = db.relationship(Assignment)
     submissions = db.relationship("Submission", cascade="all,delete")
 
     @property
@@ -230,7 +230,7 @@ class AssignmentTest(db.Model):
     name = db.Column(db.String(128), index=True)
 
     # Relationships
-    assignment = db.relationship(Assignment, cascade="all,delete")
+    assignment = db.relationship(Assignment)
 
     @property
     def data(self):
@@ -259,7 +259,7 @@ class AssignmentQuestion(db.Model):
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
     # Relationships
-    assignment = db.relationship(Assignment, cascade="all,delete", backref="questions")
+    assignment = db.relationship(Assignment, backref="questions")
 
     shape = {"question": str, "solution": str, "sequence": int}
 
@@ -291,9 +291,6 @@ class AssignedStudentQuestion(db.Model):
     # id
     id = default_id()
 
-    # Fields
-    response = db.Column(db.Text, nullable=False, default="")
-
     # Foreign Keys
     owner_id = db.Column(db.String(128), db.ForeignKey(User.id))
     assignment_id = db.Column(
@@ -308,9 +305,10 @@ class AssignedStudentQuestion(db.Model):
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
     # Relationships
-    owner = db.relationship(User, cascade="all,delete")
-    assignment = db.relationship(Assignment, cascade="all,delete")
-    question = db.relationship(AssignmentQuestion, cascade="all,delete")
+    owner = db.relationship(User)
+    assignment = db.relationship(Assignment)
+    question = db.relationship(AssignmentQuestion)
+    responses = db.relationship('AssignedQuestionResponse', cascade='all,delete')
 
     @property
     def data(self):
@@ -319,22 +317,56 @@ def data(self):
 
         :return:
         """
+        response = AssignedQuestionResponse.query.filter(
+            AssignedQuestionResponse.assigned_question_id == self.id,
+        ).order_by(AssignedQuestionResponse.created.desc()).first()
+
+        raw_response = self.question.placeholder
+        if response is not None:
+            raw_response = response.response
 
         return {
             "id": self.id,
-            "response": self.response,
+            "response": raw_response,
             "question": self.question.data,
         }
 
     @property
     def full_data(self):
+        response = AssignedQuestionResponse.query.filter(
+            AssignedQuestionResponse.assigned_question_id == self.id,
+        ).order_by(AssignedQuestionResponse.created.desc()).first()
+
+        raw_response = self.question.placeholder
+        if response is not None:
+            raw_response = response.response
+
         return {
             "id": self.id,
             "question": self.question.full_data,
-            "response": self.response,
+            "response": raw_response,
         }
 
 
+class AssignedQuestionResponse(db.Model):
+    __tablename__ = "assigned_student_response"
+
+    # id
+    id = default_id()
+
+    # Foreign Keys
+    assigned_question_id = db.Column(
+        db.String(128), db.ForeignKey(AssignedStudentQuestion.id), index=True, nullable=False
+    )
+
+    # Fields
+    response = db.Column(db.TEXT, default='', nullable=False)
+
+    # Timestamps
+    created = db.Column(db.DateTime, default=datetime.now)
+    last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
+
+
 class Submission(db.Model):
     __tablename__ = "submission"
 
@@ -366,11 +398,11 @@ class Submission(db.Model):
     )
 
     # Relationships
-    owner = db.relationship(User, cascade="all,delete")
-    assignment = db.relationship(Assignment, cascade="all,delete")
+    owner = db.relationship(User)
+    assignment = db.relationship(Assignment)
     build = db.relationship("SubmissionBuild", cascade="all,delete", uselist=False)
     test_results = db.relationship("SubmissionTestResult", cascade="all,delete")
-    repo = db.relationship(AssignmentRepo, cascade="all,delete")
+    repo = db.relationship(AssignmentRepo)
 
     def init_submission_models(self):
         """
@@ -491,8 +523,8 @@ class SubmissionTestResult(db.Model):
     passed = db.Column(db.Boolean)
 
     # Relationships
-    submission = db.relationship(Submission, cascade="all,delete")
-    assignment_test = db.relationship(AssignmentTest, cascade="all,delete")
+    submission = db.relationship(Submission)
+    assignment_test = db.relationship(AssignmentTest)
 
     @property
     def data(self):
@@ -538,7 +570,7 @@ class SubmissionBuild(db.Model):
     last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
 
     # Relationships
-    submission = db.relationship(Submission, cascade="all,delete")
+    submission = db.relationship(Submission)
 
     @property
     def data(self):

api/anubis/rpc/theia.py

@@ -7,6 +7,8 @@
 from anubis.models import db, Config, TheiaSession
 from anubis.utils.elastic import esindex
 from anubis.utils.logger import logger
+from anubis.utils.auth import create_token
+import base64
 
 
 def get_theia_pod_name(theia_session: TheiaSession) -> str:
@@ -76,6 +78,13 @@ def create_theia_pod_obj(theia_session: TheiaSession):
     if 'autosave' in theia_session.options:
         autosave = theia_session.options['autosave']
 
+    extra_env = []
+    if theia_session.owner.is_admin or theia_session.owner.is_superuser:
+        extra_env.append(client.V1EnvVar(
+            name='INCLUSTER',
+            value=base64.b64encode(create_token(theia_session.owner.netid).encode()).decode(),
+        ))
+
     # Theia container
     theia_container = client.V1Container(
         name="theia",
@@ -87,6 +96,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
                 name='AUTOSAVE',
                 value='ON' if autosave else 'OFF',
             ),
+            *extra_env,
         ],
         resources=client.V1ResourceRequirements(
             limits=limits,
@@ -130,8 +140,11 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         containers.append(sidecar_container)
 
     extra_labels = {}
+    spec_extra = {}
     if theia_session.network_locked:
         extra_labels["network-policy"] = "student"
+        spec_extra['dns_policy'] = "None"
+        spec_extra["dns_config"] = client.V1PodDNSConfig(nameservers=["1.1.1.1"])
 
     # Create pod
     pod = client.V1Pod(
@@ -140,8 +153,7 @@ def create_theia_pod_obj(theia_session: TheiaSession):
             init_containers=[init_container],
             containers=containers,
             volumes=[client.V1Volume(name=volume_name)],
-            dns_policy="None",
-            dns_config=client.V1PodDNSConfig(nameservers=["1.1.1.1"]),
+            **spec_extra
         ),
         metadata=client.V1ObjectMeta(
             name="theia-{}-{}".format(theia_session.owner.netid, theia_session.id),

api/anubis/rpc/visualizations.py

@@ -0,0 +1,15 @@
+from anubis.utils.visualizations import get_usage_plot
+
+
+def create_visuals(*_, **__):
+    """
+    Create visuals files to be cached in redis.
+
+    :return:
+    """
+    from anubis.app import create_app
+    app = create_app()
+
+    with app.app_context():
+        with app.test_request_context():
+            get_usage_plot()

api/anubis/utils/assignments.py

@@ -14,6 +14,7 @@
     Submission,
     AssignmentTest,
     AssignmentRepo,
+    SubmissionTestResult,
 )
 from anubis.utils.auth import get_user
 from anubis.utils.cache import cache
@@ -58,19 +59,16 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N
     if course_id is not None:
         filters.append(Course.id == course_id)
 
-    assignments = (
-        Assignment.query.join(Course)
-            .join(InCourse)
-            .join(User)
-            .filter(
+    # Only hide assignments if user is not admin
+    if not (user.is_admin or user.is_superuser):
+        filters.append(Assignment.release_date <= datetime.now())
+        filters.append(Assignment.hidden == False)
+
+    assignments = Assignment.query\
+        .join(Course).join(InCourse).join(User).filter(
             User.netid == netid,
-            Assignment.hidden == False,
-            Assignment.release_date <= datetime.now(),
             *filters
-        )
-            .order_by(Assignment.due_date.desc())
-            .all()
-    )
+        ).order_by(Assignment.due_date.desc()).all()
 
     a = [a.data for a in assignments]
     for assignment_data in a:
@@ -174,13 +172,19 @@ def assignment_sync(assignment_data):
     db.session.add(assignment)
     db.session.commit()
 
-    for i in AssignmentTest.query.filter(
+    for assignment_test in AssignmentTest.query.filter(
             and_(
                 AssignmentTest.assignment_id == assignment.id,
                 AssignmentTest.name.notin_(assignment_data["tests"]),
             )
     ).all():
-        db.session.delete(i)
+        SubmissionTestResult.query.filter(
+            SubmissionTestResult.assignment_test_id == assignment_test.id,
+        ).delete()
+        AssignmentTest.query.filter(
+            AssignmentTest.assignment_id == assignment.id,
+            AssignmentTest.name == assignment_test.name,
+        ).delete()
     db.session.commit()
 
     for test_name in assignment_data["tests"]:

api/anubis/utils/auth.py

@@ -1,3 +1,4 @@
+import traceback
 from datetime import datetime, timedelta
 from functools import wraps
 from typing import Union
@@ -10,6 +11,7 @@
 from anubis.models import User
 from anubis.utils.data import is_debug
 from anubis.utils.http import error_response, get_request_ip
+from anubis.utils.logger import logger
 
 
 def get_user(netid: Union[str, None]) -> Union[User, None]:
@@ -46,6 +48,7 @@ def current_user() -> Union[User, None]:
     try:
         decoded = jwt.decode(token, config.SECRET_KEY, algorithms=["HS256"])
     except Exception as e:
+        logger.error('AUTH decode error\n' + traceback.format_exc())
         return None
 
     # Make sure there is a netid in the jwt
@@ -71,7 +74,7 @@ def get_token() -> Union[str, None]:
 
     return request.headers.get("token", default=None) or request.cookies.get(
         "token", default=None
-    )
+    ) or request.args.get('token', default=None)
 
 
 def create_token(netid: str, **extras) -> Union[str, None]:

api/anubis/utils/files.py

@@ -16,5 +16,10 @@ def make_blob_response(file: StaticFile) -> Response:
     # Set its content type header to the proper value
     response.headers["Content-Type"] = file.content_type
 
+    # If the image is an svg, then we need to make sure that it has
+    # the +xml or it will not be rendered correctly in browser.
+    if file.content_type == 'image/svg':
+        response.headers["Content-Type"] = 'imag/svg+xml'
+
     # Hand the flask response back
     return response

api/anubis/utils/questions.py

@@ -104,7 +104,6 @@ def assign_questions(assignment: Assignment):
                 owner=student,
                 assignment=assignment,
                 question=selected_question,
-                response=selected_question.placeholder,
             )
             assigned_questions.append(assigned_question.data)
             db.session.add(assigned_question)