From 7a7a235b64c3500db2de63db97ce4c8ac64c9599 Mon Sep 17 00:00:00 2001 From: John McCann Cunniff Jr <36013983+wabscale@users.noreply.github.com> Date: Thu, 20 May 2021 15:04:32 -0400 Subject: [PATCH] V3.0.2 chg patch final exam (#86) * FIX remove git hook container escape * FIX remove git hook container escape * FIX migration issue, theia options for course and superuser course context * CHG improve caching issues * CHG take theia images out of deploy script * ADD app context aware with_context wrapper * CHG improve caching of visual data by forcing updates in async jobs * ADD sundial and student history visual to admin panel * CHG clean up some spacing and caching with new visuals * ADD set default course context * ADD sundial and history to admin visual tests * CHG fix perspective of in submissions get * ADD reset cache button to autograde page * CHG create repo button -> goto repo button * FIX some whoopsies * CHG much more efficent caching of data for submissions table * CHG revamp student info page * CHG rename sequence to pool * CHG put admin assignment card on seperate page * CHG ADD late assignment exceptions * CHG recalculate late exceptions and FIX sa relationship mapping * CHG late exception changed to grace date --- Makefile | 2 +- api/anubis/app.py | 2 + api/anubis/config.py | 1 + api/anubis/models/__init__.py | 116 +++---- api/anubis/rpc/batch.py | 10 +- api/anubis/rpc/pipeline.py | 81 +++-- api/anubis/rpc/seed.py | 145 +-------- api/anubis/rpc/theia.py | 301 +++++++++--------- api/anubis/rpc/visualizations.py | 20 +- api/anubis/utils/auth.py | 81 +---- api/anubis/utils/data.py | 10 + api/anubis/utils/exceptions.py | 52 +++ api/anubis/utils/http/decorators.py | 3 +- api/anubis/utils/lms/assignments.py | 72 +++-- api/anubis/utils/lms/autograde.py | 6 +- api/anubis/utils/lms/course.py | 23 +- api/anubis/utils/lms/questions.py | 49 +-- api/anubis/utils/lms/repos.py | 3 +- api/anubis/utils/lms/submissions.py | 206 ++++++++++-- api/anubis/utils/{services => lms}/theia.py | 18 +- api/anubis/utils/seed.py | 137 ++++++++ api/anubis/utils/visuals/assignments.py | 207 +++++++++++- api/anubis/utils/visuals/usage.py | 9 +- api/anubis/views/admin/__init__.py | 2 + api/anubis/views/admin/assignments.py | 53 ++- api/anubis/views/admin/autograde.py | 47 ++- api/anubis/views/admin/dangling.py | 8 +- api/anubis/views/admin/late_exceptions.py | 155 +++++++++ api/anubis/views/admin/questions.py | 4 +- api/anubis/views/admin/regrade.py | 66 +++- api/anubis/views/admin/students.py | 10 +- api/anubis/views/admin/visuals.py | 78 ++++- api/anubis/views/public/ide.py | 4 +- api/anubis/views/public/submissions.py | 67 ++-- api/anubis/views/public/visuals.py | 1 + api/anubis/views/public/webhook.py | 21 +- api/jobs/reaper.py | 49 ++- api/jobs/visuals.py | 13 +- ...da84667e_add_accept_late_to_assignments.py | 33 ++ .../716e3e14891d_add_late_exceptions.py | 44 +++ .../a622f56b9050_add_accepted_column.py | 66 ++++ ...f3ae1de1d12_chg_rename_sequence_to_pool.py | 28 ++ api/requirements.txt | 1 + api/tests/test.sh | 2 +- api/tests/test_assignment_admin.py | 8 +- api/tests/test_questions_admin.py | 10 +- api/tests/test_visuals_admin.py | 3 + k8s/chart/templates/reaper-cron.yml | 2 + k8s/chart/templates/rpc.yml | 6 + k8s/chart/templates/visuals-cron.yml | 2 + k8s/debug/provision.sh | 1 - .../admin/cli/anubis/assignment/Dockerfile | 8 +- .../admin/cli/anubis/assignment/assignment.py | 10 +- .../ide/admin/cli/anubis/assignment/meta.yml | 22 +- .../admin/cli/anubis/assignment/pipeline.py | 53 +-- .../ide/admin/cli/anubis/assignment/utils.py | 55 ++-- theia/ide/admin/cli/anubis/cli.py | 9 + web/public/index.html | 2 +- web/public/robots.txt | 9 +- .../Admin/Assignment/AssignmentCard.jsx | 26 +- .../Admin/Assignment/AssignmentReposTable.jsx | 31 ++ .../Admin/Assignment/LateExceptionAddCard.jsx | 106 ++++++ .../Admin/Assignment/RepoCommandDialog.jsx | 58 ++++ web/src/Components/Admin/Users/CourseCard.jsx | 4 +- web/src/Components/Admin/Users/UserCard.jsx | 42 +-- .../Admin/Visuals/AssignmentSundialPaper.jsx | 40 +++ .../Admin/Visuals/AutogradeVisuals.jsx | 32 +- .../Visuals/Graphs/AssignmentSundial.jsx | 135 ++++++++ .../Admin/Visuals/Graphs/StudentHistory.jsx | 160 ++++++++++ .../Visuals/StudentAssignmentHistory.jsx | 38 +++ web/src/Components/Header.jsx | 13 +- web/src/Components/NotFound.jsx | 55 ++-- .../Public/Assignments/AssignmentCard.jsx | 18 +- .../Public/Questions/QuestionsCard.jsx | 6 +- .../Public/Submissions/SubmissionsTable.jsx | 147 --------- web/src/Main.jsx | 9 +- web/src/Pages/Admin/Assignment/Assignment.jsx | 121 +++++++ .../Pages/Admin/Assignment/Assignments.jsx | 144 +++------ .../Pages/Admin/Assignment/LateExceptions.jsx | 95 ++++++ ...{AssignmentQuestions.jsx => Questions.jsx} | 1 + web/src/Pages/Admin/Assignment/Repos.jsx | 86 +++++ .../{AssignmentTests.jsx => Tests.jsx} | 2 +- .../{AutogradeResults.jsx => Assignments.jsx} | 2 +- .../{AutogradeAssignments.jsx => Results.jsx} | 34 +- ...AutogradeSubmission.jsx => Submission.jsx} | 6 +- web/src/Pages/Admin/User.jsx | 125 ++++++-- web/src/Pages/Admin/Users.jsx | 16 +- web/src/Pages/Public/Submissions.jsx | 218 ++++++++----- web/src/Pages/Public/Visuals.jsx | 30 +- web/src/Utils/datetime.js | 8 + web/src/navconfig.jsx | 38 ++- 91 files changed, 3141 insertions(+), 1211 deletions(-) create mode 100644 api/anubis/utils/exceptions.py rename api/anubis/utils/{services => lms}/theia.py (84%) create mode 100644 api/anubis/views/admin/late_exceptions.py create mode 100644 api/migrations/versions/3ed4da84667e_add_accept_late_to_assignments.py create mode 100644 api/migrations/versions/716e3e14891d_add_late_exceptions.py create mode 100644 api/migrations/versions/a622f56b9050_add_accepted_column.py create mode 100644 api/migrations/versions/bf3ae1de1d12_chg_rename_sequence_to_pool.py create mode 100644 web/src/Components/Admin/Assignment/AssignmentReposTable.jsx create mode 100644 web/src/Components/Admin/Assignment/LateExceptionAddCard.jsx create mode 100644 web/src/Components/Admin/Assignment/RepoCommandDialog.jsx create mode 100644 web/src/Components/Admin/Visuals/AssignmentSundialPaper.jsx create mode 100644 web/src/Components/Admin/Visuals/Graphs/AssignmentSundial.jsx create mode 100644 web/src/Components/Admin/Visuals/Graphs/StudentHistory.jsx create mode 100644 web/src/Components/Admin/Visuals/StudentAssignmentHistory.jsx delete mode 100644 web/src/Components/Public/Submissions/SubmissionsTable.jsx create mode 100644 web/src/Pages/Admin/Assignment/Assignment.jsx create mode 100644 web/src/Pages/Admin/Assignment/LateExceptions.jsx rename web/src/Pages/Admin/Assignment/{AssignmentQuestions.jsx => Questions.jsx} (98%) create mode 100644 web/src/Pages/Admin/Assignment/Repos.jsx rename web/src/Pages/Admin/Assignment/{AssignmentTests.jsx => Tests.jsx} (99%) rename web/src/Pages/Admin/Autograde/{AutogradeResults.jsx => Assignments.jsx} (98%) rename web/src/Pages/Admin/Autograde/{AutogradeAssignments.jsx => Results.jsx} (87%) rename web/src/Pages/Admin/Autograde/{AutogradeSubmission.jsx => Submission.jsx} (93%) create mode 100644 web/src/Utils/datetime.js diff --git a/Makefile b/Makefile index 2fa663ceb..e60c749a2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer +PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia rpc-regrade PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6 diff --git a/api/anubis/app.py b/api/anubis/app.py index 794d62a05..03ea9f651 100644 --- a/api/anubis/app.py +++ b/api/anubis/app.py @@ -13,11 +13,13 @@ def init_services(app): from anubis.utils.services.cache import cache, cache_health from anubis.utils.services.migrate import migrate from anubis.utils.services.elastic import add_global_error_handler + from anubis.utils.exceptions import add_app_exception_handlers # Init services db.init_app(app) cache.init_app(app) migrate.init_app(app, db) + add_app_exception_handlers(app) @app.route("/") def index(): diff --git a/api/anubis/config.py b/api/anubis/config.py index 501c4e85f..c29295f57 100644 --- a/api/anubis/config.py +++ b/api/anubis/config.py @@ -8,6 +8,7 @@ class Config: def __init__(self): # General flask config self.DEBUG = os.environ.get("DEBUG", default="0") == "1" + self.JOB = os.environ.get('JOB', default='0') == '1' self.SECRET_KEY = os.environ.get("SECRET_KEY", default="DEBUG") # sqlalchemy diff --git a/api/anubis/models/__init__.py b/api/anubis/models/__init__.py index 38ddf5fb0..2fb749a5b 100644 --- a/api/anubis/models/__init__.py +++ b/api/anubis/models/__init__.py @@ -49,9 +49,6 @@ class User(db.Model): created = db.Column(db.DateTime, default=datetime.now) last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now) - ta_for = db.relationship('TAForCourse', cascade='all,delete') - professor_for = db.relationship('ProfessorForCourse', cascade='all,delete') - @property def data(self): professor_for = [pf.data for pf in self.professor_for] @@ -130,7 +127,7 @@ class TAForCourse(db.Model): owner_id = db.Column(db.String(128), db.ForeignKey(User.id), primary_key=True) course_id = db.Column(db.String(128), db.ForeignKey(Course.id), primary_key=True) - owner = db.relationship(User) + owner = db.relationship(User, backref='ta_for') course = db.relationship(Course) @property @@ -148,7 +145,7 @@ class ProfessorForCourse(db.Model): owner_id = db.Column(db.String(128), db.ForeignKey(User.id), primary_key=True) course_id = db.Column(db.String(128), db.ForeignKey(Course.id), primary_key=True) - owner = db.relationship(User) + owner = db.relationship(User, backref='professor_for') course = db.relationship(Course) @property @@ -180,17 +177,18 @@ class Assignment(db.Model): course_id = db.Column(db.String(128), db.ForeignKey(Course.id), index=True) # Fields - name = db.Column(db.TEXT, nullable=False, unique=True) + name = db.Column(db.TEXT, nullable=False) hidden = db.Column(db.Boolean, default=False) description = db.Column(db.TEXT, nullable=True) github_classroom_url = db.Column(db.TEXT, nullable=True, default=None) - pipeline_image = db.Column(db.TEXT, unique=True, nullable=True) + pipeline_image = db.Column(db.TEXT, nullable=True) unique_code = db.Column( db.String(8), unique=True, default=lambda: base64.b16encode(os.urandom(4)).decode(), ) ide_enabled = db.Column(db.Boolean, default=True) + accept_late = db.Column(db.Boolean, default=True) autograde_enabled = db.Column(db.Boolean, default=True) theia_image = db.Column( db.TEXT, default="registry.osiris.services/anubis/theia-xv6" @@ -203,16 +201,23 @@ class Assignment(db.Model): grace_date = db.Column(db.DateTime, nullable=True) course = db.relationship(Course, backref="assignments") - tests = db.relationship("AssignmentTest", cascade="all,delete") - repos = db.relationship("AssignmentRepo", cascade="all,delete") + tests = db.relationship("AssignmentTest", cascade="all,delete", backref='assignment') + repos = db.relationship("AssignmentRepo", cascade="all,delete", backref='assignment') @property def data(self): + from anubis.utils.lms.assignments import get_assignment_due_date + from anubis.utils.auth import current_user + + due_date = get_assignment_due_date(current_user(), self) + return { "id": self.id, "name": self.name, - "due_date": str(self.due_date), + "due_date": str(due_date), + "past_due": due_date < datetime.now(), "hidden": self.hidden, + "accept_late": self.accept_late, "course": self.course.data, "description": self.description, "ide_enabled": self.ide_enabled, @@ -264,12 +269,11 @@ class AssignmentRepo(db.Model): # Relationships owner = db.relationship(User) - assignment = db.relationship(Assignment) - submissions = db.relationship("Submission", cascade="all,delete") @property def data(self): return { + "id": self.id, "github_username": self.github_username, "assignment_name": self.assignment.name, "course_code": self.assignment.course.course_code, @@ -290,9 +294,6 @@ class AssignmentTest(db.Model): name = db.Column(db.TEXT, index=True) hidden = db.Column(db.Boolean, default=False) - # Relationships - assignment = db.relationship(Assignment) - @property def data(self): return { @@ -314,7 +315,7 @@ class AssignmentQuestion(db.Model): # Fields question = db.Column(db.Text, nullable=False) solution = db.Column(db.Text, nullable=True) - sequence = db.Column(db.Integer, index=True, nullable=False) + pool = db.Column(db.Integer, index=True, nullable=False) code_question = db.Column(db.Boolean, default=False) code_language = db.Column(db.TEXT, nullable=True, default='') placeholder = db.Column(db.Text, nullable=True, default="") @@ -326,7 +327,7 @@ class AssignmentQuestion(db.Model): # Relationships assignment = db.relationship(Assignment, backref="questions") - shape = {"question": str, "solution": str, "sequence": int} + shape = {"question": str, "solution": str, "pool": int} @property def full_data(self): @@ -336,7 +337,7 @@ def full_data(self): "code_question": self.code_question, "code_language": self.code_language, "solution": self.solution, - "sequence": self.sequence, + "pool": self.pool, } @property @@ -346,7 +347,7 @@ def data(self): "question": self.question, "code_question": self.code_question, "code_language": self.code_language, - "sequence": self.sequence, + "pool": self.pool, } @@ -461,49 +462,14 @@ class Submission(db.Model): token = db.Column( db.String(64), default=lambda: base64.b16encode(os.urandom(32)).decode() ) + accepted = db.Column(db.Boolean, default=True) # Relationships owner = db.relationship(User) assignment = db.relationship(Assignment) - build = db.relationship("SubmissionBuild", cascade="all,delete", uselist=False) - test_results = db.relationship("SubmissionTestResult", cascade="all,delete") - repo = db.relationship(AssignmentRepo) - - def init_submission_models(self): - """ - Create adjacent submission models. - - :return: - """ - - logger.debug("initializing submission {}".format(self.id)) - - # If the models already exist, yeet - if len(self.test_results) != 0: - SubmissionTestResult.query.filter_by(submission_id=self.id).delete() - if self.build is not None: - SubmissionBuild.query.filter_by(submission_id=self.id).delete() - - # Commit deletions (if necessary) - db.session.commit() - - # Find tests for the current assignment - tests = AssignmentTest.query.filter_by(assignment_id=self.assignment_id).all() - - logger.debug("found tests: {}".format(list(map(lambda x: x.data, tests)))) - - for test in tests: - tr = SubmissionTestResult(submission_id=self.id, assignment_test_id=test.id) - db.session.add(tr) - sb = SubmissionBuild(submission_id=self.id) - db.session.add(sb) - - self.processed = False - self.state = "Waiting for resources..." - db.session.add(self) - - # Commit new models - db.session.commit() + build = db.relationship("SubmissionBuild", cascade="all,delete", backref='submission') + test_results = db.relationship("SubmissionTestResult", cascade="all,delete", backref='submission') + repo = db.relationship(AssignmentRepo, backref='submissions') @property def netid(self): @@ -614,7 +580,6 @@ class SubmissionTestResult(db.Model): passed = db.Column(db.Boolean) # Relationships - submission = db.relationship(Submission) assignment_test = db.relationship(AssignmentTest) @property @@ -660,9 +625,6 @@ class SubmissionBuild(db.Model): created = db.Column(db.DateTime, default=datetime.now) last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now) - # Relationships - submission = db.relationship(Submission) - @property def data(self): return { @@ -715,7 +677,7 @@ class TheiaSession(db.Model): @property def data(self): - from anubis.utils.services.theia import theia_redirect_url + from anubis.utils.lms.theia import theia_redirect_url return { "id": self.id, @@ -779,3 +741,31 @@ def data(self): "hidden": self.hidden, "uploaded": str(self.created) } + + +class LateException(db.Model): + user_id = db.Column(db.String(128), db.ForeignKey(User.id), primary_key=True) + assignment_id = db.Column(db.String(128), db.ForeignKey(Assignment.id), primary_key=True) + + # New Due Date + due_date = db.Column(db.DateTime, nullable=False) + + # Timestamps + created = db.Column(db.DateTime, default=datetime.now) + last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now) + + assignment = db.relationship(Assignment) + user = db.relationship(User) + + @property + def data(self): + return { + 'user_id': self.user_id, + 'user_name': self.user.name, + 'user_netid': self.user.netid, + 'assignment_id': self.assignment_id, + 'due_date': str(self.due_date) + } + + + diff --git a/api/anubis/rpc/batch.py b/api/anubis/rpc/batch.py index f23a73ddc..123f3fc3b 100644 --- a/api/anubis/rpc/batch.py +++ b/api/anubis/rpc/batch.py @@ -1,11 +1,10 @@ from anubis.utils.services.logger import logger +from anubis.utils.data import with_context +@with_context def rpc_bulk_regrade(submissions): - from anubis.app import create_app - from anubis.utils.lms.submissions import bulk_regrade_submission - - app = create_app() + from anubis.utils.lms.submissions import bulk_regrade_submissions logger.info( "bulk regrading {}".format(submissions), @@ -14,5 +13,4 @@ def rpc_bulk_regrade(submissions): }, ) - with app.app_context(): - bulk_regrade_submission(submissions) + bulk_regrade_submissions(submissions) diff --git a/api/anubis/rpc/pipeline.py b/api/anubis/rpc/pipeline.py index 02fa3b4f7..7ec23b0e4 100644 --- a/api/anubis/rpc/pipeline.py +++ b/api/anubis/rpc/pipeline.py @@ -6,7 +6,7 @@ from kubernetes import config, client from anubis.models import Config, Submission -from anubis.utils.data import is_debug +from anubis.utils.data import is_debug, with_context from anubis.utils.services.logger import logger @@ -100,6 +100,7 @@ def cleanup_jobs(batch_v1) -> int: return active_count +@with_context def create_submission_pipeline(submission_id: str): """ This function should launch the appropriate testing container @@ -107,10 +108,9 @@ def create_submission_pipeline(submission_id: str): :param submission_id: submission.id of to test """ - from anubis.app import create_app from anubis.utils.services.rpc import enqueue_autograde_pipeline + from anubis.utils.lms.submissions import init_submission - app = create_app() logger.info( "Starting submission {}".format(submission_id), @@ -119,48 +119,47 @@ def create_submission_pipeline(submission_id: str): }, ) - with app.app_context(): - max_jobs = Config.query.filter(Config.key == "MAX_JOBS").first() - max_jobs = int(max_jobs.value) if max_jobs is not None else 10 - submission = Submission.query.filter(Submission.id == submission_id).first() - - if submission is None: - logger.error( - "Unable to find submission rpc.test_repo", - extra={ - "submission_id": submission_id, - }, - ) - return - - if submission.build is None: - submission.init_submission_models() - - logger.debug( - "Found submission {}".format(submission_id), - extra={"submission": submission.data}, + max_jobs = Config.query.filter(Config.key == "MAX_JOBS").first() + max_jobs = int(max_jobs.value) if max_jobs is not None else 10 + submission = Submission.query.filter(Submission.id == submission_id).first() + + if submission is None: + logger.error( + "Unable to find submission rpc.test_repo", + extra={ + "submission_id": submission_id, + }, ) + return + + if submission.build is None: + init_submission(submission) - # Initialize kube client - config.load_incluster_config() - batch_v1 = client.BatchV1Api() + logger.debug( + "Found submission {}".format(submission_id), + extra={"submission": submission.data}, + ) - # Cleanup finished jobs - active_jobs = cleanup_jobs(batch_v1) + # Initialize kube client + config.load_incluster_config() + batch_v1 = client.BatchV1Api() - if active_jobs > max_jobs: - logger.info( - "TOO many jobs - re-enqueue {}".format(submission_id), - extra={"submission_id": submission_id}, - ) - enqueue_autograde_pipeline(submission_id) - exit(0) + # Cleanup finished jobs + active_jobs = cleanup_jobs(batch_v1) + + if active_jobs > max_jobs: + logger.info( + "TOO many jobs - re-enqueue {}".format(submission_id), + extra={"submission_id": submission_id}, + ) + enqueue_autograde_pipeline(submission_id) + exit(0) - # Create job object - job = create_pipeline_job_obj(client, submission) + # Create job object + job = create_pipeline_job_obj(client, submission) - # Log - logger.debug("creating pipeline job: " + job.to_str()) + # Log + logger.debug("creating pipeline job: " + job.to_str()) - # Send to kube api - batch_v1.create_namespaced_job(body=job, namespace="anubis") + # Send to kube api + batch_v1.create_namespaced_job(body=job, namespace="anubis") diff --git a/api/anubis/rpc/seed.py b/api/anubis/rpc/seed.py index c15f196d8..8eb7fc66b 100644 --- a/api/anubis/rpc/seed.py +++ b/api/anubis/rpc/seed.py @@ -1,6 +1,3 @@ -import random -from datetime import datetime, timedelta - from anubis.models import ( db, SubmissionTestResult, @@ -19,146 +16,22 @@ TAForCourse, ProfessorForCourse, StaticFile, + LateException, ) -from anubis.utils.data import rand, with_context +from anubis.utils.data import with_context from anubis.utils.lms.questions import assign_questions -from anubis.utils.seed import create_name, create_netid, rand_commit - - -def create_assignment(course, users): - # Assignment 1 uniq - assignment = Assignment( - id=rand(), name=f"assignment {course.name}", unique_code=rand(8), hidden=False, - pipeline_image=f"registry.osiris.services/anubis/assignment/{rand(8)}", - github_classroom_url='http://localhost', - release_date=datetime.now() - timedelta(hours=2), - due_date=datetime.now() + timedelta(hours=12), - grace_date=datetime.now() + timedelta(hours=13), - course_id=course.id, ide_enabled=True, autograde_enabled=False, - ) - - for i in range(random.randint(2, 4)): - b, c = random.randint(1, 5), random.randint(1, 5) - assignment_question = AssignmentQuestion( - id=rand(), - question=f"What is {c} + {b}?", - solution=f"{c + b}", - sequence=i, - code_question=False, - assignment_id=assignment.id, - ) - db.session.add(assignment_question) - - tests = [] - for i in range(random.randint(3, 5)): - tests.append(AssignmentTest(id=rand(), name=f"test {i}", assignment_id=assignment.id)) - - submissions = [] - repos = [] - theia_sessions = [] - for user in users: - repos.append( - AssignmentRepo( - id=rand(), owner=user, assignment_id=assignment.id, - repo_url="https://github.com/wabscale/xv6-public", - github_username=user.github_username, - ) - ) - - # for _ in range(2): - # theia_sessions.append( - # TheiaSession( - # owner=user, - # assignment=assignment, - # repo_url=repos[-1].repo_url, - # active=False, - # ended=datetime.now(), - # state="state", - # cluster_address="127.0.0.1", - # ) - # ) - # theia_sessions.append( - # TheiaSession( - # owner=user, - # assignment=assignment, - # repo_url=repos[-1].repo_url, - # active=True, - # state="state", - # cluster_address="127.0.0.1", - # ) - # ) - - if random.randint(0, 3) != 0: - for i in range(random.randint(1, 10)): - submissions.append( - Submission( - id=rand(), - commit=rand_commit(), - state="Waiting for resources...", - owner=user, - assignment_id=assignment.id, - repo=repos[-1], - ) - ) - - db.session.add_all(tests) - db.session.add_all(submissions) - db.session.add_all(theia_sessions) - db.session.add_all(repos) - db.session.add(assignment) - - return assignment, tests, submissions, repos - - -def create_students(n=10): - students = [] - for i in range(random.randint(n // 2, n)): - name = create_name() - netid = create_netid(name) - students.append( - User( - name=name, - netid=netid, - github_username=rand(8), - is_superuser=False, - ) - ) - db.session.add_all(students) - return students - - -def create_course(users, **kwargs): - course = Course(id=rand(), **kwargs) - db.session.add(course) - - for user in users: - db.session.add(InCourse(owner=user, course=course)) - - return course - - -def init_submissions(submissions): - # Init models - for submission in submissions: - submission.init_submission_models() - submission.processed = True - - build_pass = random.randint(0, 2) != 0 - submission.build.passed = build_pass - submission.build.stdout = 'blah blah blah build' - - if build_pass: - for test_result in submission.test_results: - test_passed = random.randint(0, 3) != 0 - test_result.passed = test_passed - - test_result.message = 'Test passed' if test_passed else 'Test failed' - test_result.stdout = 'blah blah blah test output' +from anubis.utils.seed import ( + create_assignment, + create_students, + create_course, + init_submissions, +) @with_context def seed(): # Yeet + LateException.query.delete() TheiaSession.query.delete() AssignedQuestionResponse.query.delete() AssignedStudentQuestion.query.delete() diff --git a/api/anubis/rpc/theia.py b/api/anubis/rpc/theia.py index 4be717482..cbd4e1ff7 100644 --- a/api/anubis/rpc/theia.py +++ b/api/anubis/rpc/theia.py @@ -6,6 +6,7 @@ from kubernetes import config, client from anubis.models import db, Config, TheiaSession +from anubis.utils.data import with_context from anubis.utils.auth import create_token from anubis.utils.services.elastic import esindex from anubis.utils.services.logger import logger @@ -173,6 +174,7 @@ def create_theia_pod_obj(theia_session: TheiaSession): return pod, pvc +@with_context def initialize_theia_session(theia_session_id: str): """ Create the kube resources for a theia session. Will update database entries if necessary. @@ -180,9 +182,7 @@ def initialize_theia_session(theia_session_id: str): :param theia_session_id: :return: """ - from anubis.app import create_app - app = create_app() config.load_incluster_config() v1 = client.CoreV1Api() @@ -193,88 +193,87 @@ def initialize_theia_session(theia_session_id: str): }, ) - with app.app_context(): - max_ides = Config.query.filter(Config.key == "MAX_IDES").first() - max_ides = int(max_ides.value) if max_ides is not None else 10 - theia_session = TheiaSession.query.filter( - TheiaSession.id == theia_session_id, - ).first() + max_ides = Config.query.filter(Config.key == "MAX_IDES").first() + max_ides = int(max_ides.value) if max_ides is not None else 10 + theia_session = TheiaSession.query.filter( + TheiaSession.id == theia_session_id, + ).first() - if TheiaSession.query.filter(TheiaSession.active == True).count() >= max_ides: - # If there are too many active pods, recycle the job through the - # queue - logger.info( - "Maximum IDEs currently running. Re-enqueuing session_id={} initialized request".format( - theia_session_id - ) + if TheiaSession.query.filter(TheiaSession.active == True).count() >= max_ides: + # If there are too many active pods, recycle the job through the + # queue + logger.info( + "Maximum IDEs currently running. Re-enqueuing session_id={} initialized request".format( + theia_session_id ) - from anubis.utils.services.rpc import enqueue_ide_initialize - - enqueue_ide_initialize(theia_session_id) - return + ) + from anubis.utils.services.rpc import enqueue_ide_initialize - if theia_session is None: - logger.error( - "Unable to find theia session rpc.initialize_theia_session", - extra={"theia_session_id": theia_session_id}, - ) - return + enqueue_ide_initialize(theia_session_id) + return - logger.debug( - "Found theia_session {}".format(theia_session_id), - extra={"submission": theia_session.data}, + if theia_session is None: + logger.error( + "Unable to find theia session rpc.initialize_theia_session", + extra={"theia_session_id": theia_session_id}, ) + return + + logger.debug( + "Found theia_session {}".format(theia_session_id), + extra={"submission": theia_session.data}, + ) - # Create pod, and pvc object - pod, pvc = create_theia_pod_obj(theia_session) + # Create pod, and pvc object + pod, pvc = create_theia_pod_obj(theia_session) - # Log - logger.info("creating theia pod: " + pod.to_str()) + # Log + logger.info("creating theia pod: " + pod.to_str()) - # Send to kube api - v1.create_namespaced_persistent_volume_claim(namespace="anubis", body=pvc) - v1.create_namespaced_pod(namespace="anubis", body=pod) + # Send to kube api + v1.create_namespaced_persistent_volume_claim(namespace="anubis", body=pvc) + v1.create_namespaced_pod(namespace="anubis", body=pod) - # Wait for it to have started, then update theia_session state - name = get_theia_pod_name(theia_session) - n = 10 - while True: - pod: client.V1Pod = v1.read_namespaced_pod( - name=name, - namespace="anubis", - ) + # Wait for it to have started, then update theia_session state + name = get_theia_pod_name(theia_session) + n = 10 + while True: + pod: client.V1Pod = v1.read_namespaced_pod( + name=name, + namespace="anubis", + ) - if pod.status.phase == "Pending": - n += 1 - if n > 60: - logger.error( - "Theia session took too long to initialize. Freeing worker." - ) - break - - time.sleep(1) - - if pod.status.phase == "Running": - theia_session.cluster_address = pod.status.pod_ip - theia_session.state = "Running" - esindex( - "theia", - body={ - "event": "session-init", - "session_id": theia_session.id, - "netid": theia_session.owner.netid, - }, + if pod.status.phase == "Pending": + n += 1 + if n > 60: + logger.error( + "Theia session took too long to initialize. Freeing worker." ) - logger.info("Theia session started {}".format(name)) break - if pod.status.phase == "Failed": - theia_session.active = False - theia_session.state = "Failed" - logger.error("Theia session failed {}".format(name)) - break + time.sleep(1) - db.session.commit() + if pod.status.phase == "Running": + theia_session.cluster_address = pod.status.pod_ip + theia_session.state = "Running" + esindex( + "theia", + body={ + "event": "session-init", + "session_id": theia_session.id, + "netid": theia_session.owner.netid, + }, + ) + logger.info("Theia session started {}".format(name)) + break + + if pod.status.phase == "Failed": + theia_session.active = False + theia_session.state = "Failed" + logger.error("Theia session failed {}".format(name)) + break + + db.session.commit() def reap_theia_session_resources(theia_session_id: str): @@ -380,128 +379,120 @@ def check_active_pods(): db.session.commit() +@with_context def reap_theia_session(theia_session_id: str): - from anubis.app import create_app - - app = create_app() config.load_incluster_config() logger.info("Attempting to reap theia session {}".format(theia_session_id)) - with app.app_context(): - theia_session: TheiaSession = TheiaSession.query.filter( - TheiaSession.id == theia_session_id, - ).first() + theia_session: TheiaSession = TheiaSession.query.filter( + TheiaSession.id == theia_session_id, + ).first() - if theia_session is None: - logger.error( - "Could not find theia session {} when attempting to delete".format( - theia_session_id - ) + if theia_session is None: + logger.error( + "Could not find theia session {} when attempting to delete".format( + theia_session_id ) - return + ) + return - reap_theia_session_resources(theia_session_id) + reap_theia_session_resources(theia_session_id) - if theia_session.active: - theia_session.active = False - theia_session.ended = datetime.now() + if theia_session.active: + theia_session.active = False + theia_session.ended = datetime.now() - theia_session.state = "Ended" - db.session.commit() + theia_session.state = "Ended" + db.session.commit() +@with_context def reap_all_theia_sessions(course_id: str): - from anubis.app import create_app - - app = create_app() config.load_incluster_config() logger.info("Clearing theia sessions") - with app.app_context(): - theia_sessions = TheiaSession.query.filter( - TheiaSession.active == True, - TheiaSession.course_id == course_id, - ).all() - - for n, theia_session in enumerate(theia_sessions): - # Get pod name - name = get_theia_pod_name(theia_session) - - if theia_session.active: - # Log deletion - logger.info( - "deleting theia session pod: {}".format(name), - extra={"session": theia_session.data}, - ) + theia_sessions = TheiaSession.query.filter( + TheiaSession.active == True, + TheiaSession.course_id == course_id, + ).all() + + for n, theia_session in enumerate(theia_sessions): + # Get pod name + name = get_theia_pod_name(theia_session) + + if theia_session.active: + # Log deletion + logger.info( + "deleting theia session pod: {}".format(name), + extra={"session": theia_session.data}, + ) - # Reap kube resources - reap_theia_session_resources(theia_session.id) + # Reap kube resources + reap_theia_session_resources(theia_session.id) - # Update the database row - theia_session.active = False - theia_session.state = "Ended" - theia_session.ended = datetime.now() + # Update the database row + theia_session.active = False + theia_session.state = "Ended" + theia_session.ended = datetime.now() - # Batch commits in size of 5 - if n % 5 == 0: - db.session.commit() + # Batch commits in size of 5 + if n % 5 == 0: + db.session.commit() - db.session.commit() + db.session.commit() +@with_context def reap_stale_theia_sessions(*_): - from anubis.app import create_app from anubis.config import config as _config - app = create_app() config.load_incluster_config() v1 = client.CoreV1Api() logger.info("Clearing stale theia sessions") - with app.app_context(): - resp = v1.list_namespaced_pod( - namespace="anubis", label_selector="app.kubernetes.io/name=theia,role=theia-session" - ) + resp = v1.list_namespaced_pod( + namespace="anubis", label_selector="app.kubernetes.io/name=theia,role=theia-session" + ) - for n, pod in enumerate(resp.items): - session_id = pod.metadata.labels["session"] - theia_session: TheiaSession = TheiaSession.query.filter( - TheiaSession.id == session_id - ).first() - theia_session.cluster_address = pod.status.pod_ip + for n, pod in enumerate(resp.items): + session_id = pod.metadata.labels["session"] + theia_session: TheiaSession = TheiaSession.query.filter( + TheiaSession.id == session_id + ).first() + theia_session.cluster_address = pod.status.pod_ip - # Make sure we have a session to work on - if theia_session is None: - continue + # Make sure we have a session to work on + if theia_session is None: + continue - # If the session is younger than 6 hours old, continue - if datetime.now() <= theia_session.created + _config.THEIA_TIMEOUT: - logger.info(f'NOT reaping session {theia_session.id}') - continue + # If the session is younger than 6 hours old, continue + if datetime.now() <= theia_session.created + _config.THEIA_TIMEOUT: + logger.info(f'NOT reaping session {theia_session.id}') + continue - # Log deletion - logger.info( - "REAPING theia session pod: {}".format(session_id), - extra={"session": theia_session.data}, - ) + # Log deletion + logger.info( + "REAPING theia session pod: {}".format(session_id), + extra={"session": theia_session.data}, + ) - # Reap - reap_theia_session_resources(theia_session.id) + # Reap + reap_theia_session_resources(theia_session.id) - # Update the database row - theia_session.active = False - theia_session.state = "Ended" - theia_session.ended = datetime.now() + # Update the database row + theia_session.active = False + theia_session.state = "Ended" + theia_session.ended = datetime.now() - # Batch commits in size of 5 - if n % 5 == 0: - db.session.commit() + # Batch commits in size of 5 + if n % 5 == 0: + db.session.commit() - # Make sure that database entries marked as active have pods - # and pods have active database entries - check_active_pods() + # Make sure that database entries marked as active have pods + # and pods have active database entries + check_active_pods() - db.session.commit() + db.session.commit() diff --git a/api/anubis/rpc/visualizations.py b/api/anubis/rpc/visualizations.py index 883db1db2..34827a1f1 100644 --- a/api/anubis/rpc/visualizations.py +++ b/api/anubis/rpc/visualizations.py @@ -1,15 +1,25 @@ +from datetime import datetime + +from anubis.models import Assignment +from anubis.config import config +from anubis.utils.data import with_context +from anubis.utils.visuals.assignments import get_assignment_sundial from anubis.utils.visuals.usage import get_usage_plot +@with_context def create_visuals(*_, **__): """ Create visuals files to be cached in redis. :return: """ - from anubis.app import create_app - app = create_app() + get_usage_plot() + + recent_assignments = Assignment.query.filter( + Assignment.release_date > datetime.now(), + Assignment.due_date > datetime.now() - config.STATS_REAP_DURATION, + ).all() - with app.app_context(): - with app.test_request_context(): - get_usage_plot() + for assignment in recent_assignments: + get_assignment_sundial(assignment.id) diff --git a/api/anubis/utils/auth.py b/api/anubis/utils/auth.py index d8f0f40b9..ed79f5d31 100644 --- a/api/anubis/utils/auth.py +++ b/api/anubis/utils/auth.py @@ -11,35 +11,11 @@ from anubis.config import config from anubis.models import User, TAForCourse, ProfessorForCourse from anubis.utils.data import is_debug +from anubis.utils.exceptions import AuthenticationError, LackCourseContext from anubis.utils.http.https import error_response, get_request_ip from anubis.utils.services.logger import logger -class AuthenticationError(Exception): - """ - This exception should be raised if a request - lacks the proper authentication fow whatever - action they are requesting. - - If the view function is wrapped in any of the - require auth decorators, then this exception - will be caught and return a 401. - """ - - -class LackCourseContext(Exception): - """ - Most of the admin actions require there to - be a course context to be set. This exception - should be raised if there is not a course - context set. - - If there is some other permission issue involving - the course context, then a AuthenticationError - may be more appropriate. - """ - - def get_user(netid: Union[str, None]) -> Union[User, None]: """ Load a user by username @@ -126,49 +102,6 @@ def create_token(netid: str, **extras) -> Union[str, None]: }, config.SECRET_KEY) -def _course_context_wrapper(function): - """ - Wrap a view function or view decorator with a - LackCourseContext handler. This should be applied - to the admin require decorators. - - :param function: - :return: - """ - - @functools.wraps(function) - def wrapper(*args, **kwargs): - try: - return function(*args, **kwargs) - except LackCourseContext as e: - logger.error(traceback.format_exc()) - return error_response(str(e) or 'Please set your course context') - - return wrapper - - -def _auth_context_wrapper(function): - """ - Wrap a view function or decorator with an - AuthenticationError handler. It will handle - the exception with a 401. This should be - applied to all the require decorators. - - :param function: - :return: - """ - - @functools.wraps(function) - def wrapper(*args, **kwargs): - try: - return function(*args, **kwargs) - except AuthenticationError as e: - logger.error(traceback.format_exc()) - return error_response(str(e) or 'Unauthenticated'), 401 - - return wrapper - - def require_user(unless_debug=False): """ Wrap a function to require a user to be logged in. @@ -181,8 +114,6 @@ def require_user(unless_debug=False): def decorator(func): @wraps(func) - @_auth_context_wrapper - @_course_context_wrapper def wrapper(*args, **kwargs): # Get the user in the current # request context. @@ -197,7 +128,7 @@ def wrapper(*args, **kwargs): # in the current request context, and # that use is an admin. if user is None: - return error_response("Unauthenticated"), 401 + raise AuthenticationError() # Pass the parameters to the # decorated function. @@ -221,8 +152,6 @@ def require_admin(unless_debug=False, unless_vpn=False): def decorator(func): @wraps(func) - @_auth_context_wrapper - @_course_context_wrapper def wrapper(*args, **kwargs): # Get the user in the current # request context. @@ -276,8 +205,6 @@ def require_superuser(unless_debug=False, unless_vpn=False): def decorator(func): @wraps(func) - @_auth_context_wrapper - @_course_context_wrapper def wrapper(*args, **kwargs): # Get the user in the current # request context. @@ -296,12 +223,12 @@ def wrapper(*args, **kwargs): # in the current request context, and # that use is a superuser. if user is None: - return error_response("Unauthenticated"), 401 + raise AuthenticationError() # If the user is not a superuser, then return a 400 error # so it will be displayed in a snackbar. if user.is_superuser is False: - return error_response("Requires superuser") + raise AuthenticationError("Requires superuser") # Pass the parameters to the # decorated function. diff --git a/api/anubis/utils/data.py b/api/anubis/utils/data.py index 2fed45cbb..427ea79cb 100644 --- a/api/anubis/utils/data.py +++ b/api/anubis/utils/data.py @@ -21,6 +21,16 @@ def is_debug() -> bool: return config.DEBUG +def is_job() -> bool: + """ + Returns true if the app context is used in a job. + + :return: + """ + + return config.JOB + + def jsonify(data, status_code=200): """ Wrap a data response to set proper headers for json diff --git a/api/anubis/utils/exceptions.py b/api/anubis/utils/exceptions.py new file mode 100644 index 000000000..4cd1cd42d --- /dev/null +++ b/api/anubis/utils/exceptions.py @@ -0,0 +1,52 @@ +import traceback + +from flask import Flask, jsonify + + +class AuthenticationError(Exception): + """ + This exception should be raised if a request + lacks the proper authentication fow whatever + action they are requesting. + + If the view function is wrapped in any of the + require auth decorators, then this exception + will be caught and return a 401. + """ + + +class LackCourseContext(Exception): + """ + Most of the admin actions require there to + be a course context to be set. This exception + should be raised if there is not a course + context set. + + If there is some other permission issue involving + the course context, then a AuthenticationError + may be more appropriate. + """ + + +def add_app_exception_handlers(app: Flask): + """ + Add exception handlers to the flask app. + + :param app: + :return: + """ + + from anubis.utils.http.https import error_response + from anubis.utils.services.logger import logger + + # Set AuthenticationError handler + @app.errorhandler(AuthenticationError) + def handler_authentication_error(e: AuthenticationError): + logger.error(traceback.format_exc()) + return jsonify(error_response(str(e) or 'Unauthenticated')), 401 + + # Set LackCourseContext handler + @app.errorhandler(LackCourseContext) + def handle_lack_course_context(e: LackCourseContext): + logger.error(traceback.format_exc()) + return error_response(str(e) or 'Please set your course context') diff --git a/api/anubis/utils/http/decorators.py b/api/anubis/utils/http/decorators.py index 702a26d7c..750cbe350 100644 --- a/api/anubis/utils/http/decorators.py +++ b/api/anubis/utils/http/decorators.py @@ -5,7 +5,8 @@ from flask import request from anubis.models import Submission -from anubis.utils.auth import current_user, AuthenticationError +from anubis.utils.auth import current_user +from anubis.utils.exceptions import AuthenticationError from anubis.utils.data import jsonify, _verify_data_shape from anubis.utils.http.https import error_response diff --git a/api/anubis/utils/lms/assignments.py b/api/anubis/utils/lms/assignments.py index 5c0abd0c9..356858890 100644 --- a/api/anubis/utils/lms/assignments.py +++ b/api/anubis/utils/lms/assignments.py @@ -1,6 +1,6 @@ import traceback from datetime import datetime -from typing import Union, List, Dict, Tuple +from typing import Union, List, Dict, Tuple, Optional from dateutil.parser import parse as date_parse, ParserError from sqlalchemy import or_ @@ -15,6 +15,7 @@ AssignmentTest, AssignmentRepo, SubmissionTestResult, + LateException, ) from anubis.utils.auth import get_user from anubis.utils.data import is_debug @@ -40,7 +41,7 @@ def get_courses(netid: str): return [c.data for c in classes] -@cache.memoize(timeout=10, unless=is_debug) +@cache.memoize(timeout=10, unless=is_debug, source_check=True) def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], None]: """ Get all the current assignments for a netid. Optionally specify a class_name @@ -72,20 +73,20 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N # Build a list of all the assignments visible # to this user for each of the specified courses. assignments: List[Assignment] = [] - for course in course_ids: + for _course_id in course_ids: # Query filters filters = [] # If the current user is not a course admin or a superuser, then # we should filter out assignments that have not been released, # and those marked as hidden. - if not (user.is_superuser or is_course_admin(course_id)): + if not is_course_admin(_course_id, user_id=user.id): filters.append(Assignment.release_date <= datetime.now()) filters.append(Assignment.hidden == False) # Get the assignment objects that should be visible to this user. course_assignments = Assignment.query.join(Course).filter( - Course.id == course, + Course.id == _course_id, *filters, ).all() @@ -105,18 +106,24 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N for assignment_data in response: # If the current user has a submission for this assignment, then mark it assignment_data["has_submission"] = ( - Submission.query.join(User).join(Assignment).filter( - Assignment.id == assignment_data["id"], - User.netid == netid, - ).first() is not None + Submission.query.join(User).join(Assignment).filter( + Assignment.id == assignment_data["id"], + User.netid == netid, + ).first() is not None ) + repo = AssignmentRepo.query.filter( + AssignmentRepo.owner_id == user.id, + AssignmentRepo.assignment_id == assignment_data['id'], + ).first() + # If the current user has a repo for this assignment, then mark it assignment_data["has_repo"] = ( - AssignmentRepo.query.filter( - AssignmentRepo.owner_id == user.id, - AssignmentRepo.assignment_id == assignment_data['id'], - ).first() is not None + repo is not None + ) + # If the current user has a repo for this assignment, then mark it + assignment_data["repo_url"] = ( + repo.repo_url if repo is not None else None ) return response @@ -197,14 +204,10 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]: for test_name in assignment_data["tests"]: # Find if the assignment test exists - assignment_test = ( - AssignmentTest.query.filter( - Assignment.id == assignment.id, - AssignmentTest.name == test_name, - ) - .join(Assignment) - .first() - ) + assignment_test = AssignmentTest.query.join(Assignment).filter( + Assignment.id == assignment.id, + AssignmentTest.name == test_name, + ).first() # Create the assignment test if it did not already exist if assignment_test is None: @@ -224,3 +227,30 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]: return {"assignment": assignment.data, "questions": question_message}, True +def get_assignment_due_date(user: Optional[User], assignment: Assignment) -> datetime: + """ + Get the due date for an assignment for a specific user. We check to + see if there is a late exception for this user, and return that if + available. Otherwise, the default due date for the assignment is + returned. + + :param user: + :param assignment: + :return: + """ + + if user is None: + return assignment.grace_date + + # Check for a late exception for this student + late_exception: Optional[LateException] = LateException.query.filter( + LateException.user_id == user.id, + LateException.assignment_id == assignment.id, + ).first() + + # If there was a late exception, return that due_date + if late_exception is not None: + return late_exception.due_date + + # If no late exception, return the assignment default + return assignment.grace_date diff --git a/api/anubis/utils/lms/autograde.py b/api/anubis/utils/lms/autograde.py index 526d99e64..d8e2e72c9 100644 --- a/api/anubis/utils/lms/autograde.py +++ b/api/anubis/utils/lms/autograde.py @@ -36,7 +36,8 @@ def autograde(student_id, assignment_id): Submission.query.filter( Submission.assignment_id == assignment_id, Submission.owner_id == student_id, - Submission.processed, + Submission.processed == True, + Submission.accepted == True, ) .order_by(Submission.created.desc()) .all() @@ -81,8 +82,10 @@ def autograde_submission_result_wrapper(assignment: Assignment, user_id: str, ne "netid": netid, "name": name, "submission": None, + "build_passed": False, "tests_passed": 0, "total_tests": 0, + "tests_passed_names": [], "full_stats": None, "master": None, "commits": None, @@ -105,6 +108,7 @@ def autograde_submission_result_wrapper(assignment: Assignment, user_id: str, ne "build_passed": submission.build.passed if submission.build is not None else False, "tests_passed": best_count, "total_tests": len(submission.test_results), + "tests_passed_names": [test.assignment_test.name for test in submission.test_results if test.passed], "full_stats": "https://anubis.osiris.services/api/private/submission/{}".format( submission.id ), diff --git a/api/anubis/utils/lms/course.py b/api/anubis/utils/lms/course.py index e8566159a..f7236e2dc 100644 --- a/api/anubis/utils/lms/course.py +++ b/api/anubis/utils/lms/course.py @@ -6,7 +6,8 @@ from flask import request -from anubis.utils.auth import LackCourseContext, current_user, AuthenticationError +from anubis.utils.auth import current_user +from anubis.utils.exceptions import AuthenticationError, LackCourseContext from anubis.utils.services.logger import logger from anubis.models import ( Course, @@ -22,6 +23,7 @@ StaticFile, User, InCourse, + LateException, ) @@ -107,17 +109,22 @@ def _get_course_context(): return context -def is_course_superuser(course_id: str) -> bool: +def is_course_superuser(course_id: str, user_id: str = None) -> bool: """ Use this function to verify if the current user is a superuser for the specified course_id. A user is a superuser for a course if they are a professor, or if they are a superuser. :param course_id: + :param user_id: :return: """ + # Get the current user - user = current_user() + if user_id is None: + user = current_user() + else: + user = User.query.filter(User.id == user_id).first() # If they are a superuser, then we can just return True if user.is_superuser: @@ -133,17 +140,22 @@ def is_course_superuser(course_id: str) -> bool: return prof is not None -def is_course_admin(course_id: str) -> bool: +def is_course_admin(course_id: str, user_id: str = None) -> bool: """ Use this function to verify if the current user is an admin for the specified course_id. A user is an admin for a course if they are a ta, professor, or if they are a superuser. :param course_id: + :param user_id: :return: """ + # Get the current user - user = current_user() + if user_id is None: + user = current_user() + else: + user = User.query.filter(User.id == user_id).first() # If they are a superuser, then just return True if user.is_superuser: @@ -249,6 +261,7 @@ def assert_course_context(*models: Tuple[Any]): AssignmentRepo, AssignedStudentQuestion, AssignmentQuestion, + LateException, ]: if isinstance(model, model_type): object_stack.append(model.assignment) diff --git a/api/anubis/utils/lms/questions.py b/api/anubis/utils/lms/questions.py index b0e36d404..71d4ee4a3 100644 --- a/api/anubis/utils/lms/questions.py +++ b/api/anubis/utils/lms/questions.py @@ -13,7 +13,7 @@ from anubis.utils.services.cache import cache -def get_question_sequence_mapping( +def get_question_pool_mapping( questions: List[AssignmentQuestion], ) -> Dict[int, List[AssignmentQuestion]]: """ @@ -31,12 +31,12 @@ def get_question_sequence_mapping( """ # Get unique sequences - sequences = set(question.sequence for question in questions) + pools = set(question.pool for question in questions) # Build up sequence to question mapping - sequence_to_questions = {sequence: [] for sequence in sequences} + sequence_to_questions = {pool: [] for pool in pools} for question in questions: - sequence_to_questions[question.sequence].append(question) + sequence_to_questions[question.pool].append(question) return sequence_to_questions @@ -85,7 +85,7 @@ def assign_questions(assignment: Assignment): AssignmentQuestion.assignment_id == assignment.id, ).all() - questions = get_question_sequence_mapping(raw_questions) + questions = get_question_pool_mapping(raw_questions) # Go through students in the class and assign them questions assigned_questions = [] @@ -159,7 +159,7 @@ def ingest_questions(questions: dict, assignment: Assignment): ) continue - sequence = question_sequence["sequence"] + pool = question_sequence["pool"] for question in question_sequence["questions"]: # Check to see if question already exists for the current @@ -178,7 +178,7 @@ def ingest_questions(questions: dict, assignment: Assignment): assignment_id=assignment.id, question=question["q"], solution=question["a"], - sequence=sequence, + pool=pool, ) db.session.add(assignment_question) accepted.append({"question": question}) @@ -189,19 +189,18 @@ def ingest_questions(questions: dict, assignment: Assignment): return accepted, ignored, rejected -def get_all_questions(assignment: Assignment) -> Dict[int, List[Dict[str, str]]]: +def get_all_questions(assignment: Assignment) -> List[Dict[str, str]]: """ Get all questions for a given assignment. - response = { - 1 : [ - { - question: "what is 2*2?", - solution: "4" - }, + response = [ + { + question: "what is 2*2?", + solution: "4", ... - ] - } + }, + ... + ] :param assignment: :return: @@ -213,13 +212,19 @@ def get_all_questions(assignment: Assignment) -> Dict[int, List[Dict[str, str]]] ).all() # Get sequence to question mapping - sequence_to_questions = get_question_sequence_mapping(questions) + pools_to_questions = get_question_pool_mapping(questions) - # Convert ORM object to a dictionary - return { - _sequence: [_question.full_data for _question in _questions] - for _sequence, _questions in sequence_to_questions.items() - } + # Get the raw questions in a generator of lists + question_pools = pools_to_questions.values() + + # Pull questions out of sequences and add + # them to question_list + questions_list: List[Dict[str, str]] = [] + for pool in question_pools: + for q in pool: + questions_list.append(q.full_data) + + return questions_list @cache.memoize(timeout=5, unless=is_debug) diff --git a/api/anubis/utils/lms/repos.py b/api/anubis/utils/lms/repos.py index 4f7dd021f..0808fbde8 100644 --- a/api/anubis/utils/lms/repos.py +++ b/api/anubis/utils/lms/repos.py @@ -2,9 +2,10 @@ from anubis.models import AssignmentRepo, Assignment from anubis.utils.services.cache import cache +from anubis.utils.data import is_debug -@cache.memoize(timeout=3600) +@cache.memoize(timeout=3600, source_check=True, unless=is_debug) def get_repos(user_id: str): repos: List[AssignmentRepo] = ( AssignmentRepo.query.join(Assignment) diff --git a/api/anubis/utils/lms/submissions.py b/api/anubis/utils/lms/submissions.py index dc7ab531e..9cf40150c 100644 --- a/api/anubis/utils/lms/submissions.py +++ b/api/anubis/utils/lms/submissions.py @@ -1,14 +1,29 @@ from datetime import datetime -from typing import List, Union, Dict - -from anubis.models import Submission, AssignmentRepo, User, db, Course, Assignment, InCourse -from anubis.utils.data import is_debug +from typing import List, Union, Dict, Optional, Tuple + +from anubis.models import ( + db, + User, + Course, + Submission, + AssignmentRepo, + SubmissionTestResult, + AssignmentTest, + SubmissionBuild, + Assignment, + InCourse, + LateException, +) +from anubis.utils.data import is_debug, split_chunks from anubis.utils.http.https import error_response, success_response from anubis.utils.services.cache import cache -from anubis.utils.services.rpc import enqueue_autograde_pipeline +from anubis.utils.services.rpc import rpc_enqueue, enqueue_autograde_pipeline +from anubis.utils.lms.assignments import get_assignment_due_date +from anubis.rpc.batch import rpc_bulk_regrade +from anubis.utils.services.logger import logger -def bulk_regrade_submission(submissions: List[Submission]) -> List[dict]: +def bulk_regrade_submissions(submissions: List[Submission]) -> List[dict]: """ Regrade a batch of submissions :param submissions: @@ -57,7 +72,7 @@ def regrade_submission(submission: Union[Submission, str], queue: str = 'default submission.last_updated = datetime.now() # Reset the accompanying database objects - submission.init_submission_models() + init_submission(submission) # Enqueue the submission job enqueue_autograde_pipeline(submission.id, queue=queue) @@ -104,26 +119,35 @@ def fix_dangling(): # Find all the submissions that belong to that # repo, fix then grade them. - for s in dangling_repo.submissions: + for submission in dangling_repo.submissions: # Give the submission an owner - s.owner_id = owner.id - db.session.add(s) + submission.owner_id = owner.id + db.session.add(submission) db.session.commit() # Update running tally of fixed submissions - fixed.append(s.data) + fixed.append(submission.data) - # Enqueue a autograde job for the submission - enqueue_autograde_pipeline(s.id) + # Get the due date + due_date = get_assignment_due_date(owner, dangling_repo.assignment) + + # Check if the submission should be accepted + if dangling_repo.assignment.accept_late and submission.created < due_date: + # Enqueue a autograde job for the submission + enqueue_autograde_pipeline(submission.id) + + # Reject the submission if it was late + else: + reject_late_submission(submission) # Find dangling submissions dangling_submissions = Submission.query.filter(Submission.owner_id == None).all() # Iterate through all submissions lacking an owner - for s in dangling_submissions: + for submission in dangling_submissions: # Try to find a repo to match dangling_repo = AssignmentRepo.query.filter( - AssignmentRepo.id == s.assignment_repo_id + AssignmentRepo.id == submission.assignment_repo_id ).first() # Try to find an owner student @@ -137,27 +161,38 @@ def fix_dangling(): db.session.commit() # Give the submission an owner - s.owner_id = owner.id - db.session.add(s) + submission.owner_id = owner.id + db.session.add(submission) db.session.commit() # Update running tally of fixed submissions - fixed.append(s.data) + fixed.append(submission.data) + + # Get the due date + due_date = get_assignment_due_date(owner, submission.assignment) + + # Check if the submission should be accepted + if submission.assignment.accept_late and submission.created < due_date: + # Enqueue a autograde job for the submission + enqueue_autograde_pipeline(submission.id) - # Enqueue a autograde job for the submission - enqueue_autograde_pipeline(s.id) + # Reject the submission if it was late + else: + reject_late_submission(submission) return fixed -@cache.memoize(timeout=3600, unless=is_debug) +@cache.memoize(timeout=5, unless=is_debug, source_check=True) def get_submissions( - user_id=None, course_id=None, assignment_id=None -) -> Union[List[Dict[str, str]], None]: + user_id=None, course_id=None, assignment_id=None, limit=None, offset=None, +) -> Optional[Tuple[List[Dict[str, str]], int]]: """ Get all submissions for a given netid. Cache the results. Optionally specify a class_name and / or assignment_name for additional filtering. + :param offset: + :param limit: :param user_id: :param course_id: :param assignment_id: id of assignment @@ -180,13 +215,132 @@ def get_submissions( if assignment_id is not None: filters.append(Assignment.id == assignment_id) - submissions = ( + + query = ( Submission.query.join(Assignment) .join(Course) .join(InCourse) .join(User) .filter(Submission.owner_id == owner.id, *filters) - .all() + + .order_by(Submission.created.desc()) ) - return [s.full_data for s in submissions] \ No newline at end of file + all_total = query.count() + + if limit is not None: + query = query.limit(limit) + if offset is not None: + query = query.offset(offset) + + submissions = query.all() + + return [s.full_data for s in submissions], all_total + + +def recalculate_late_submissions(student: User, assignment: Assignment): + """ + Recalculate the submissions that need to be + switched from accepted to rejected. + + :param student: + :param assignment: + :return: + """ + + # Get the due date for this student + due_date = get_assignment_due_date(student, assignment) + + # Get the submissions that need to be rejected + s_reject = Submission.query.filter( + Submission.created > due_date, + Submission.accepted == True, + ).all() + + # Get the submissions that need to be accepted + s_accept = Submission.query.filter( + Submission.created < due_date, + Submission.accepted == False, + ).all() + + # Go through, and reset and enqueue regrade + s_accept_ids = list(map(lambda x: x.id, s_accept)) + for chunk in split_chunks(s_accept_ids, 32): + rpc_enqueue(rpc_bulk_regrade, 'regrade', args=[chunk]) + + # Reject the submissions that need to be updated + for submission in s_reject: + reject_late_submission(submission) + + # Commit the changes + db.session.commit() + + +def reject_late_submission(submission: Submission): + """ + Set all the fields that need to be set when + rejecting a submission. + + * Does not commit changes * + + :return: + """ + + # Go through test results, and set them to rejected + for test_result in submission.test_results: + test_result: SubmissionTestResult + test_result.passed = False + test_result.message = 'Late submissions not accepted' + test_result.stdout = '' + db.session.add(test_result) + + # Go through build results, and set them to rejected + submission.build.passed = False + submission.build.stdout = 'Late submissions not accepted' + db.session.add(submission.build) + + # Set the fields on self to be rejected + submission.accepted = False + submission.processed = True + submission.state = "Late submissions not accepted" + db.session.add(submission) + + +def init_submission(submission: Submission, commit: bool = True): + """ + Create adjacent submission models. + + :return: + """ + + logger.debug("initializing submission {}".format(submission.id)) + + # If the models already exist, yeet + if len(submission.test_results) != 0: + SubmissionTestResult.query.filter_by(submission_id=submission.id).delete() + if submission.build is not None: + SubmissionBuild.query.filter_by(submission_id=submission.id).delete() + + if commit: + # Commit deletions (if necessary) + db.session.commit() + + # Find tests for the current assignment + tests = AssignmentTest.query.filter_by(assignment_id=submission.assignment_id).all() + + logger.debug("found tests: {}".format(list(map(lambda x: x.data, tests)))) + + for test in tests: + tr = SubmissionTestResult(submission_id=submission.id, assignment_test_id=test.id) + db.session.add(tr) + sb = SubmissionBuild(submission_id=submission.id) + db.session.add(sb) + + submission.accepted = True + submission.processed = False + submission.state = "Waiting for resources..." + db.session.add(submission) + + if commit: + # Commit new models + db.session.commit() diff --git a/api/anubis/utils/services/theia.py b/api/anubis/utils/lms/theia.py similarity index 84% rename from api/anubis/utils/services/theia.py rename to api/anubis/utils/lms/theia.py index 3b1fa67ba..7c5e57e22 100644 --- a/api/anubis/utils/services/theia.py +++ b/api/anubis/utils/lms/theia.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union +from typing import List, Tuple, Union, Dict from werkzeug.utils import redirect @@ -9,6 +9,22 @@ from anubis.utils.services.cache import cache +@cache.memoize(timeout=5, source_check=True) +def get_recent_sessions(user_id: str, limit: int = 10, offset: int = 10) -> List[Dict]: + student = User.query.filter( + User.id == user_id, + ).first() + + sessions: List[TheiaSession] = TheiaSession.query.filter( + TheiaSession.owner_id == student.id, + ).order_by(TheiaSession.created.desc()).limit(limit).offset(offset).all() + + return [ + session.data + for session in sessions + ] + + @cache.memoize(timeout=5, unless=is_debug) def get_n_available_sessions() -> Tuple[int, int]: """ diff --git a/api/anubis/utils/seed.py b/api/anubis/utils/seed.py index 6f7b63e77..b21d06549 100644 --- a/api/anubis/utils/seed.py +++ b/api/anubis/utils/seed.py @@ -1,5 +1,10 @@ import random import string +from datetime import datetime, timedelta + +from anubis.models import Assignment, AssignmentQuestion, db, AssignmentTest, AssignmentRepo, Submission, User, Course, \ + InCourse +from anubis.utils.data import rand names = ["Joette", "Anabelle", "Fred", "Woodrow", "Neoma", "Dorian", "Treasure", "Tami", "Berdie", "Jordi", "Frances", "Gerhardt", "Kristina", "Carmelita", "Sim", "Hideo", "Arland", "Wirt", "Robt", "Narcissus", "Steve", "Monique", @@ -63,3 +68,135 @@ def rand_commit(n=40) -> str: from anubis.utils.data import rand return rand(n) + + +def create_assignment(course, users): + # Assignment 1 uniq + assignment = Assignment( + id=rand(), name=f"assignment {course.name}", unique_code=rand(8), hidden=False, + pipeline_image=f"registry.osiris.services/anubis/assignment/{rand(8)}", + github_classroom_url='http://localhost', + release_date=datetime.now() - timedelta(hours=2), + due_date=datetime.now() + timedelta(hours=12), + grace_date=datetime.now() + timedelta(hours=13), + course_id=course.id, ide_enabled=True, autograde_enabled=False, + ) + + for i in range(random.randint(2, 4)): + b, c = random.randint(1, 5), random.randint(1, 5) + assignment_question = AssignmentQuestion( + id=rand(), + question=f"What is {c} + {b}?", + solution=f"{c + b}", + pool=i, + code_question=False, + assignment_id=assignment.id, + ) + db.session.add(assignment_question) + + tests = [] + for i in range(random.randint(3, 5)): + tests.append(AssignmentTest(id=rand(), name=f"test {i}", assignment_id=assignment.id)) + + submissions = [] + repos = [] + theia_sessions = [] + for user in users: + repos.append( + AssignmentRepo( + id=rand(), owner=user, assignment_id=assignment.id, + repo_url="https://github.com/wabscale/xv6-public", + github_username=user.github_username, + ) + ) + + # for _ in range(2): + # theia_sessions.append( + # TheiaSession( + # owner=user, + # assignment=assignment, + # repo_url=repos[-1].repo_url, + # active=False, + # ended=datetime.now(), + # state="state", + # cluster_address="127.0.0.1", + # ) + # ) + # theia_sessions.append( + # TheiaSession( + # owner=user, + # assignment=assignment, + # repo_url=repos[-1].repo_url, + # active=True, + # state="state", + # cluster_address="127.0.0.1", + # ) + # ) + + if random.randint(0, 3) != 0: + for i in range(random.randint(1, 10)): + submission = Submission( + id=rand(), + commit=rand_commit(), + state="Waiting for resources...", + owner=user, + assignment_id=assignment.id, + ) + submission.repo = repos[-1] + submissions.append(submission) + + db.session.add_all(tests) + db.session.add_all(submissions) + db.session.add_all(theia_sessions) + db.session.add_all(repos) + db.session.add(assignment) + + return assignment, tests, submissions, repos + + +def create_students(n=10): + students = [] + for i in range(random.randint(n // 2, n)): + name = create_name() + netid = create_netid(name) + students.append( + User( + name=name, + netid=netid, + github_username=rand(8), + is_superuser=False, + ) + ) + db.session.add_all(students) + return students + + +def create_course(users, **kwargs): + course = Course(id=rand(), **kwargs) + db.session.add(course) + + for user in users: + db.session.add(InCourse(owner=user, course=course)) + + return course + + +def init_submissions(submissions): + from anubis.utils.lms.submissions import init_submission + + # Init models + for submission in submissions: + init_submission(submission) + submission.processed = True + + build_pass = random.randint(0, 2) != 0 + submission.build.passed = build_pass + submission.build.stdout = 'blah blah blah build' + + if build_pass: + for test_result in submission.test_results: + test_passed = random.randint(0, 3) != 0 + test_result.passed = test_passed + + test_result.message = 'Test passed' if test_passed else 'Test failed' + test_result.stdout = 'blah blah blah test output' diff --git a/api/anubis/utils/visuals/assignments.py b/api/anubis/utils/visuals/assignments.py index b90109a50..4364c2924 100644 --- a/api/anubis/utils/visuals/assignments.py +++ b/api/anubis/utils/visuals/assignments.py @@ -2,9 +2,19 @@ import pandas as pd from typing import List, Any, Union, Dict -from anubis.models import db, AssignmentTest -from anubis.utils.data import is_debug +from anubis.models import ( + db, + AssignmentTest, + Assignment, + User, + TheiaSession, + Submission, + SubmissionBuild, + SubmissionTestResult +) +from anubis.utils.data import is_debug, is_job from anubis.utils.services.cache import cache +from anubis.utils.lms.autograde import bulk_autograde from anubis.utils.visuals.queries import ( time_to_pass_test_sql, assignment_test_fail_nosub_sql, @@ -119,3 +129,196 @@ def get_assignment_tests_pass_counts(assignment_test: AssignmentTest): {'label': 'test failed', 'theta': fail_count, 'color': 'red'}, {'label': 'test passed', 'theta': pass_count, 'color': 'green'}, ] + + +@cache.memoize(timeout=60, unless=is_debug, source_check=True) +def get_assignment_history(assignment_id, netid): + """ + + :param assignment_id: + :param netid: + :return: + """ + + assignment = Assignment.query.filter( + Assignment.id == assignment_id + ).first() + other = User.query.filter(User.netid == netid).first() + + db_theia_sessions = TheiaSession.query.filter( + TheiaSession.owner_id == other.id, + TheiaSession.assignment_id == assignment.id + ).all() + + db_submissions = Submission.query.filter( + Submission.assignment_id == assignment.id, + Submission.owner_id == other.id, + ).order_by(Submission.created.desc()).all() + + test_count = len(assignment.full_data['tests']) + + test_results = [] + build_results = [] + for db_submission in db_submissions: + created = db_submission.created.replace(microsecond=0, second=0) + build_passed = 1 if db_submission.build.passed else 0 + tests_passed = sum(map(lambda test: (1 if test['result']['passed'] else 0), db_submission.all_tests)) + + test_results.append({ + 'x': str(created), + 'y': tests_passed, + 'total': test_count, + 'label': f'{tests_passed}/{test_count} tests passed' + }) + + build_results.append({ + 'x': str(created), + 'y': build_passed, + 'label': 'build passed' if build_passed == 1 else 'build failed' + }) + + return { + 'submissions': { + 'test_results': test_results, + 'build_results': build_results, + }, + 'dates': { + 'release_date': [{'x': str(assignment.release_date), 'y': test_count}], + 'due_date': [{'x': str(assignment.due_date), 'y': test_count}], + 'grace_date': [{'x': str(assignment.grace_date), 'y': test_count}], + } + } + + +@cache.memoize(timeout=-1, source_check=True, forced_update=is_job) +def get_assignment_sundial(assignment_id): + """ + Get the sundial data for a specific assignment. The basic breakdown of + this data is: + + submission -> test -> {passed, failed} + + :param assignment_id: + :return: + """ + + # Get the assignment + assignment = Assignment.query.filter( + Assignment.id == assignment_id + ).first() + + # Create base sundial + sundial = { + 'children': [ + + # Build Passed + { + 'name': 'build passed', + 'hex': '#8b0eea', + 'children': [ + {'name': test.name, 'hex': '#004080', 'children': [ + # Test Passed + {'name': 'passed', 'hex': '#008000', 'value': 0}, + + # Test Failed + {'name': 'failed', 'hex': '#800000', 'value': 0}, + ]} + for test in assignment.tests + ], + }, + + # Build Failed + { + 'name': 'build failed', + 'hex': '#f00', + 'value': 0, + }, + + # No Submission + { + 'name': 'no submission', + 'hex': '#808080', + 'value': 0, + }, + ] + } + + # Get the autograde results for the entire assignment. This + # function call is cached. + autograde_results = bulk_autograde(assignment_id, offset=0, limit=300) + + # Count the number of build and no submissions to + # insert into the name label. + build_passed = 0 + build_failed = 0 + no_submission = 0 + + # Go through all the autograde results + for result in autograde_results: + + # If there was no submission, then increment no submission + if result['submission'] is None: + no_submission += 1 + sundial['children'][2]['value'] += 1 + continue + + # If the build passed, go through the tests + if result['build_passed']: + build_passed += 1 + + # Set of tests passed names + tests_passed = set(result['tests_passed_names']) + + for index in range(len(sundial['children'][0]['children'])): + # Get the test name + test_name = sundial['children'][0]['children'][index]['name'] + + # If this student passed this test, then increment the tests passed value + if test_name in tests_passed: + sundial['children'][0]['children'][index]['children'][0]['value'] += 1 + + # If this student failed this test, then increment the tests failed value + else: + sundial['children'][0]['children'][index]['children'][1]['value'] += 1 + + continue + + # If the build failed, then skip the tests + if not result['build_passed']: + + # If the student failed the build, then we increment the failed build value + build_failed += 1 + sundial['children'][1]['value'] += 1 + + continue + + # Update the title for the high level names + sundial['children'][0]['name'] = f'{build_passed} builds passed' + sundial['children'][1]['name'] = f'{build_failed} builds failed' + sundial['children'][2]['name'] = f'{no_submission} no submissions' + + # Update the title for the individual tests + for test in sundial['children'][0]['children']: + passed = test['children'][0] + failed = test['children'][1] + passed_value = passed['value'] + failed_value = failed['value'] + + passed['name'] = f'{passed_value} passed' + failed['name'] = f'{failed_value} failed' + + return sundial + + + + + + + + + + + + + + diff --git a/api/anubis/utils/visuals/usage.py b/api/anubis/utils/visuals/usage.py index df1a901b4..af4e57b78 100644 --- a/api/anubis/utils/visuals/usage.py +++ b/api/anubis/utils/visuals/usage.py @@ -6,8 +6,9 @@ import pandas as pd from anubis.models import Assignment, Submission, TheiaSession -from anubis.utils.data import is_debug +from anubis.utils.data import is_job from anubis.utils.services.cache import cache +from anubis.utils.services.logger import logger def get_submissions() -> pd.DataFrame: @@ -76,7 +77,7 @@ def get_theia_sessions() -> pd.DataFrame: # Drop outliers based on duration theia_sessions = theia_sessions[ np.abs(theia_sessions.duration - theia_sessions.duration.mean()) <= (3 * theia_sessions.duration.std()) - ] + ] return theia_sessions @@ -132,11 +133,13 @@ def get_raw_submissions() -> List[Dict[str, Any]]: # return df -@cache.memoize(timeout=420, unless=is_debug) +@cache.memoize(timeout=-1, forced_update=is_job) def get_usage_plot(): import matplotlib.pyplot as plt import matplotlib.colors as mcolors + logger.info('GENERATING USAGE PLOT PNG') + assignments = Assignment.query.filter( Assignment.hidden == False, Assignment.release_date <= datetime.now(), diff --git a/api/anubis/views/admin/__init__.py b/api/anubis/views/admin/__init__.py index af2dbbf96..5bbe469ba 100644 --- a/api/anubis/views/admin/__init__.py +++ b/api/anubis/views/admin/__init__.py @@ -12,6 +12,7 @@ def register_admin_views(app): from anubis.views.admin.config import config_ from anubis.views.admin.visuals import visuals_ from anubis.views.admin.dangling import dangling + from anubis.views.admin.late_exceptions import late_exceptions_ views = [ ide, @@ -27,6 +28,7 @@ def register_admin_views(app): config_, visuals_, dangling, + late_exceptions_, ] for view in views: diff --git a/api/anubis/views/admin/assignments.py b/api/anubis/views/admin/assignments.py index 54d9b9a93..10e265b84 100644 --- a/api/anubis/views/admin/assignments.py +++ b/api/anubis/views/admin/assignments.py @@ -1,17 +1,25 @@ import json +import parse from dateutil.parser import parse as dateparse from flask import Blueprint from sqlalchemy.exc import DataError, IntegrityError -from anubis.models import db, Assignment, User, AssignmentTest, SubmissionTestResult +from anubis.models import ( + db, + Assignment, + AssignmentRepo, + User, + AssignmentTest, + SubmissionTestResult, +) from anubis.utils.auth import require_admin from anubis.utils.data import rand from anubis.utils.data import row2dict from anubis.utils.http.decorators import load_from_id, json_response, json_endpoint from anubis.utils.http.https import error_response, success_response from anubis.utils.lms.assignments import assignment_sync -from anubis.utils.lms.course import assert_course_admin, get_course_context, assert_course_context +from anubis.utils.lms.course import get_course_context, assert_course_context from anubis.utils.lms.questions import get_assigned_questions from anubis.utils.services.elastic import log_endpoint from anubis.utils.services.logger import logger @@ -19,6 +27,42 @@ assignments = Blueprint("admin-assignments", __name__, url_prefix="/admin/assignments") +@assignments.route('/repos/') +@require_admin() +@load_from_id(Assignment, verify_owner=False) +@json_response +def admin_assignments_repos_id(assignment: Assignment): + """ + + :param assignment: + :return: + """ + + assert_course_context(assignment) + + repos = AssignmentRepo.query.filter( + AssignmentRepo.assignment_id == assignment.id, + ).all() + + def get_ssh_url(url): + r = parse.parse('https://github.com/{}', url) + path = r[0] + path = path.removesuffix('.git') + return f'git@github.com:{path}.git' + + return success_response({'assignment': assignment.full_data, 'repos': [ + { + 'id': repo.id, + 'url': repo.repo_url, + 'ssh': get_ssh_url(repo.repo_url), + 'github_username': repo.github_username, + 'name': repo.owner.name if repo.owner_id is not None else 'N/A', + 'netid': repo.owner.netid if repo.owner_id is not None else 'N/A', + } + for repo in repos + ]}) + + @assignments.route("/assignment//questions/get/") @require_admin() @log_endpoint("cli", lambda: "question get") @@ -64,7 +108,10 @@ def admin_assignments_get_id(assignment: Assignment): assert_course_context(assignment) # Pass back the full data - return success_response({"assignment": assignment.full_data}) + return success_response({ + "assignment": row2dict(assignment), + "tests": [test.data for test in assignment.tests], + }) @assignments.route("/list") diff --git a/api/anubis/views/admin/autograde.py b/api/anubis/views/admin/autograde.py index d8bbd8bfc..3615b9ef6 100644 --- a/api/anubis/views/admin/autograde.py +++ b/api/anubis/views/admin/autograde.py @@ -1,19 +1,58 @@ from flask import Blueprint -from anubis.models import Submission, Assignment, User, InCourse +from anubis.models import Submission, Assignment, User from anubis.utils.auth import require_admin from anubis.utils.http.decorators import json_response from anubis.utils.http.https import success_response, error_response, get_number_arg from anubis.utils.lms.autograde import bulk_autograde, autograde, autograde_submission_result_wrapper -from anubis.utils.lms.course import assert_course_admin, assert_course_context, get_course_context +from anubis.utils.lms.course import assert_course_context from anubis.utils.lms.questions import get_assigned_questions -from anubis.utils.services.elastic import log_endpoint from anubis.utils.services.cache import cache +from anubis.utils.services.elastic import log_endpoint +from anubis.utils.visuals.assignments import ( + get_admin_assignment_visual_data, + get_assignment_history, + get_assignment_sundial, +) autograde_ = Blueprint("admin-autograde", __name__, url_prefix="/admin/autograde") -@autograde_.route("/assignment/") +@autograde_.route('/cache-reset/') +@require_admin() +@cache.memoize(timeout=60) +@json_response +def admin_autograde_cache_reset(assignment_id: str): + """ + Clear the autograde cache for a specific assignment. + + :param assignment_id: + :return: + """ + # Pull the assignment object + assignment = Assignment.query.filter( + Assignment.id == assignment_id + ).first() + + # Verify that we got an assignment + if assignment is None: + return error_response('assignment does not exist') + + # Verify that the current course context, and the assignment course match + assert_course_context(assignment) + + cache.delete_memoized(bulk_autograde) + cache.delete_memoized(autograde) + cache.delete_memoized(get_assignment_history) + cache.delete_memoized(get_admin_assignment_visual_data) + cache.delete_memoized(get_assignment_sundial) + + return success_response({ + 'message': 'success' + }) + + +@autograde_.route("/assignment/") @require_admin() @cache.memoize(timeout=60) @json_response diff --git a/api/anubis/views/admin/dangling.py b/api/anubis/views/admin/dangling.py index 51a3f5d47..474dbc65a 100644 --- a/api/anubis/views/admin/dangling.py +++ b/api/anubis/views/admin/dangling.py @@ -4,7 +4,7 @@ from anubis.utils.auth import require_superuser from anubis.utils.http.decorators import json_response from anubis.utils.http.https import success_response -from anubis.utils.lms.submissions import fix_dangling +from anubis.utils.lms.submissions import fix_dangling, init_submission from anubis.utils.services.elastic import log_endpoint dangling = Blueprint("admin-dangling", __name__, url_prefix="/admin/dangling") @@ -50,12 +50,12 @@ def private_reset_dangling(): resets = [] # Iterate over all the dangling submissions - for s in Submission.query.filter_by(owner_id=None).all(): + for submission in Submission.query.filter_by(owner_id=None).all(): # Reset the submission models - s.init_submission_models() + init_submission(submission) # Append the new dangling submission data for the response - resets.append(s.data) + resets.append(submission.data) # Return all the reset submissions return success_response({"reset": resets}) diff --git a/api/anubis/views/admin/late_exceptions.py b/api/anubis/views/admin/late_exceptions.py new file mode 100644 index 000000000..ef8db17b8 --- /dev/null +++ b/api/anubis/views/admin/late_exceptions.py @@ -0,0 +1,155 @@ +from typing import Optional + +from flask import Blueprint +from dateutil.parser import parse as date_parse, ParserError + +from anubis.models import db, LateException, Assignment, User +from anubis.utils.auth import require_admin +from anubis.utils.http.decorators import json_response, json_endpoint +from anubis.utils.http.https import success_response, error_response +from anubis.utils.lms.course import assert_course_context +from anubis.utils.lms.submissions import recalculate_late_submissions + + +late_exceptions_ = Blueprint('admin-late-exceptions', __name__, url_prefix='/admin/late-exceptions') + + +@late_exceptions_.route('/list/') +@require_admin() +@json_response +def admin_late_exception_list(assignment_id: str): + """ + List all late exceptions for an assignment + + :param assignment_id: + :return: + """ + + # Get the assignment + assignment = Assignment.query.filter( + Assignment.id == assignment_id, + ).first() + + # Make sure it exists + if assignment is None: + return error_response('assignment does not exist') + + # Assert the course context + assert_course_context(assignment) + + # Get late exceptions + late_exceptions = LateException.query.filter( + LateException.assignment_id == assignment.id + ).all() + + # Break down for response + return success_response({'assignment': assignment.full_data, 'late_exceptions': [ + late_exception.data + for late_exception in late_exceptions + ]}) + + +@late_exceptions_.post('/update') +@require_admin() +@json_endpoint([('assignment_id', str), ('user_id', str), ('due_date', str)]) +def admin_late_exception_update(assignment_id: str = None, user_id: str = None, due_date: str = None): + """ + Add or update a late exception + + :param due_date: + :param user_id: + :param assignment_id: + :return: + """ + + # Get the assignment and user + assignment = Assignment.query.filter( + Assignment.id == assignment_id, + ).first() + student = User.query.filter(User.id == user_id).first() + + # Make sure assignment and user exist + if assignment is None: + return error_response('assignment does not exist') + if student is None: + return error_response('user does not exist') + + assert_course_context(assignment, student) + + # Get late exceptions + late_exception: Optional[LateException] = LateException.query.filter( + LateException.assignment_id == assignment.id, + LateException.user_id == student.id, + ).first() + if late_exception is None: + late_exception = LateException( + assignment_id=assignment.id, + user_id=student.id, + ) + db.session.add(late_exception) + + # Try to parse the datetime + try: + due_date = date_parse(due_date) + except ParserError: + return error_response('datetime could not be parsed') + + if due_date < assignment.due_date: + return error_response('Exception cannot be before assignment due date') + + # Update the due date + late_exception.due_date = due_date + + db.session.commit() + + # Recalculate the late submissions + recalculate_late_submissions(student, assignment) + + # Break down for response + return success_response({ + 'status': 'Late exceptions updated' + }) + + +@late_exceptions_.route('/remove//') +@require_admin() +@json_response +def admin_late_exception_remove(assignment_id: str = None, user_id: str = None): + """ + Add or update a late exception + + :param user_id: + :param assignment_id: + :return: + """ + + # Get the assignment and user + assignment = Assignment.query.filter( + Assignment.id == assignment_id, + ).first() + student = User.query.filter(User.id == user_id).first() + + # Make sure assignment and user exist + if assignment is None: + return error_response('assignment does not exist') + if student is None: + return error_response('user does not exist') + + assert_course_context(assignment, student) + + # Delete the exception if it exists + LateException.query.filter( + LateException.assignment_id == assignment.id, + LateException.user_id == student.id, + ).delete() + + # Recalculate the late submissions + recalculate_late_submissions(student, assignment) + + db.session.commit() + + # Break down for response + return success_response({ + 'status': 'Late exception deleted', + 'variant': 'warning', + }) diff --git a/api/anubis/views/admin/questions.py b/api/anubis/views/admin/questions.py index f7515c4bc..9f3d34b0f 100644 --- a/api/anubis/views/admin/questions.py +++ b/api/anubis/views/admin/questions.py @@ -48,7 +48,7 @@ def admin_questions_add_unique_code(unique_code: str): # Create a new, blank question aq = AssignmentQuestion( assignment_id=assignment.id, - sequence=0, + pool=0, question='', solution='', code_question=False, @@ -286,7 +286,7 @@ def private_questions_get_assignments_unique_code(unique_code: str): # Get all the question assignments assignment_questions = AssignmentQuestion.query.filter( AssignmentQuestion.assignment_id == assignment.id, - ).order_by(AssignmentQuestion.sequence, AssignmentQuestion.created.desc()).all() + ).order_by(AssignmentQuestion.pool, AssignmentQuestion.created.desc()).all() return success_response({ 'questions': [ diff --git a/api/anubis/views/admin/regrade.py b/api/anubis/views/admin/regrade.py index 1e05dbd0a..b0f685725 100644 --- a/api/anubis/views/admin/regrade.py +++ b/api/anubis/views/admin/regrade.py @@ -4,7 +4,7 @@ from flask import Blueprint from sqlalchemy import or_ -from anubis.models import Submission, Assignment +from anubis.models import Submission, Assignment, User from anubis.rpc.batch import rpc_bulk_regrade from anubis.utils.auth import require_admin from anubis.utils.data import split_chunks @@ -13,7 +13,10 @@ from anubis.utils.http.https import error_response, success_response, get_number_arg from anubis.utils.lms.course import assert_course_context from anubis.utils.services.elastic import log_endpoint +from anubis.utils.services.cache import cache from anubis.utils.services.rpc import enqueue_autograde_pipeline, rpc_enqueue +from anubis.utils.lms.autograde import bulk_autograde, autograde +from anubis.utils.lms.submissions import init_submission regrade = Blueprint("admin-regrade", __name__, url_prefix="/admin/regrade") @@ -84,7 +87,7 @@ def admin_regrade_submission_commit(commit: str): assert_course_context(submission) # Reset submission in database - submission.init_submission_models() + init_submission(submission) # Enqueue the submission pipeline enqueue_autograde_pipeline(submission.id) @@ -93,6 +96,65 @@ def admin_regrade_submission_commit(commit: str): return success_response({"submission": submission.data, "user": submission.owner.data}) +@regrade.route("/student//") +@require_admin() +@log_endpoint("cli", lambda: "regrade") +@json_response +def private_regrade_student_assignment_netid(assignment_id: str, netid: str): + """ + + :param assignment_id: + :param netid: + :return: + """ + + # Find the assignment + assignment: Assignment = Assignment.query.filter( + or_(Assignment.id == assignment_id, Assignment.name == assignment_id) + ).first() + + # Verify that the assignment exists + if assignment is None: + return error_response("cant find assignment") + + # Get the student + student: User = User.query.filter( + User.netid == netid + ).first() + + # Verify the student exists + if student is None: + return error_response('Student does not exist') + + # Assert that the course exists + assert_course_context(student, assignment) + + submissions = Submission.query.filter( + Submission.assignment_id == assignment.id, + Submission.owner_id == student.id, + ).all() + + # Get a count of submissions for the response + submission_count = len(submissions) + + # Split the submissions into bite sized chunks + submission_ids = [s.id for s in submissions] + submission_chunks = split_chunks(submission_ids, 100) + + # Enqueue each chunk as a job for the rpc workers + for chunk in submission_chunks: + rpc_enqueue(rpc_bulk_regrade, 'regrade', args=[chunk]) + + # Clear cache of autograde results + cache.delete_memoized(bulk_autograde, assignment.id) + cache.delete_memoized(autograde, student.id, assignment.id) + + return success_response({ + "status": f"{submission_count} submissions enqueued. This may take a while.", + "submissions": submission_ids, + }) + + @regrade.route("/assignment/") @require_admin() @log_endpoint("cli", lambda: "regrade") diff --git a/api/anubis/views/admin/students.py b/api/anubis/views/admin/students.py index 5a5cad0b9..a74b3da72 100644 --- a/api/anubis/views/admin/students.py +++ b/api/anubis/views/admin/students.py @@ -6,6 +6,9 @@ from anubis.utils.http.https import success_response, error_response, get_number_arg from anubis.utils.lms.course import assert_course_superuser, get_course_context, assert_course_context from anubis.utils.lms.students import get_students +from anubis.utils.lms.repos import get_repos +from anubis.utils.lms.theia import get_recent_sessions + students_ = Blueprint("admin-students", __name__, url_prefix="/admin/students") @@ -90,10 +93,15 @@ def admin_students_info_id(id: str): Course.id.in_(course_ids) ).all() + repos = get_repos(student.id) + recent_theia = get_recent_sessions(student.id) + # Pass back the student and course information return success_response({ - "student": student.data, + "user": student.data, "courses": [course.data for course in courses], + "repos": repos, + "theia": recent_theia, }) diff --git a/api/anubis/views/admin/visuals.py b/api/anubis/views/admin/visuals.py index 0730188f7..1f4781638 100644 --- a/api/anubis/views/admin/visuals.py +++ b/api/anubis/views/admin/visuals.py @@ -1,11 +1,15 @@ from flask import Blueprint -from anubis.models import Assignment +from anubis.models import Assignment, User from anubis.utils.auth import require_admin from anubis.utils.http.decorators import json_response from anubis.utils.http.https import success_response, error_response -from anubis.utils.visuals.assignments import get_admin_assignment_visual_data -from anubis.utils.lms.course import get_course_context, assert_course_context, assert_course_admin +from anubis.utils.lms.course import assert_course_context +from anubis.utils.visuals.assignments import ( + get_admin_assignment_visual_data, + get_assignment_history, + get_assignment_sundial, +) visuals_ = Blueprint('admin-visuals', __name__, url_prefix='/admin/visuals') @@ -42,3 +46,71 @@ def public_visuals_assignment_id(assignment_id: str): assignment_id ) }) + + +@visuals_.route('/history//') +@require_admin() +@json_response +def visual_history_assignment_netid(assignment_id: str, netid: str): + """ + Get the visual history for a specific student and assignment. + + * lightly cached per assignment and user * + + :param assignment_id: + :param netid: + :return: + """ + + # Get the assignment object + assignment = Assignment.query.filter( + Assignment.id == assignment_id + ).first() + + # If the assignment does not exist, then stop + if assignment is None: + return error_response('Assignment does not exist') + + # Get the student + student = User.query.filter(User.netid == netid).first() + + # Make sure that the student exists + if student is None: + return error_response('netid does not exist') + + # Assert that both the course and the assignment are + # within the view of the current admin. + assert_course_context(student, assignment) + + # Get tha cached assignment history + return success_response(get_assignment_history(assignment.id, student.netid)) + + +@visuals_.route('/sundial/') +@require_admin() +@json_response +def visual_sundial_assignment(assignment_id: str): + """ + Get the summary sundial data for an assignment. This endpoint + is ridiculously IO intensive. + + * heavily cached * + + :param assignment_id: + :return: + """ + # Get the assignment object + assignment = Assignment.query.filter( + Assignment.id == assignment_id + ).first() + + # If the assignment does not exist, then stop + if assignment is None: + return error_response('Assignment does not exist') + + # Assert that the assignment is within the view of + # the current admin. + assert_course_context(assignment) + + # Pull the (maybe cached) sundial data + return success_response({'sundial': get_assignment_sundial(assignment.id)}) diff --git a/api/anubis/views/public/ide.py b/api/anubis/views/public/ide.py index a6568c944..ced91ebba 100644 --- a/api/anubis/views/public/ide.py +++ b/api/anubis/views/public/ide.py @@ -12,7 +12,7 @@ from anubis.utils.services.elastic import log_endpoint from anubis.utils.services.logger import logger from anubis.utils.services.rpc import enqueue_ide_stop, enqueue_ide_initialize -from anubis.utils.services.theia import ( +from anubis.utils.lms.theia import ( theia_redirect_url, get_n_available_sessions, theia_poll_ide, @@ -176,7 +176,7 @@ def public_ide_initialize(assignment: Assignment): {"active": active_session.active, "session": active_session.data} ) - if user.is_superuser or is_course_admin(assignment.course_id): + if not is_course_admin(assignment.course_id): if datetime.now() <= assignment.release_date: return error_response("Assignment has not been released.") diff --git a/api/anubis/views/public/submissions.py b/api/anubis/views/public/submissions.py index 518db2d5c..9cd0ae1a3 100644 --- a/api/anubis/views/public/submissions.py +++ b/api/anubis/views/public/submissions.py @@ -1,9 +1,12 @@ +from datetime import datetime + from flask import Blueprint, request -from anubis.models import User, Submission +from anubis.models import User, Submission, Assignment from anubis.utils.auth import current_user, require_user from anubis.utils.http.decorators import json_response -from anubis.utils.http.https import error_response, success_response +from anubis.utils.http.https import error_response, success_response, get_number_arg +from anubis.utils.lms.course import is_course_admin, assert_course_context from anubis.utils.lms.submissions import regrade_submission, get_submissions from anubis.utils.services.elastic import log_endpoint from anubis.utils.services.logger import logger @@ -30,31 +33,47 @@ def public_submissions(): :return: """ + # Get optional filters course_id = request.args.get("courseId", default=None) perspective_of_id = request.args.get("userId", default=None) assignment_id = request.args.get("assignmentId", default=None) + # Get the limit and offset for submissions query + limit: int = get_number_arg('limit', default_value=10) + offset: int = get_number_arg('offset', default_value=0) + # Load current user user: User = current_user() - - if perspective_of_id is not None and not (user.is_superuser): + perspective_of = user + if perspective_of_id is not None: + perspective_of = User.query.filter(User.id == perspective_of_id).first() + + # If the request is from the perspective of a different user, + # we need to make sure the requester is an admin in the current + # course context. + if perspective_of_id is not None and not is_course_admin(course_id): return error_response("Bad Request"), 400 - logger.debug("id: " + str(perspective_of_id)) - logger.debug("id: " + str(perspective_of_id or user.id)) - - submissions_ = get_submissions( + # Get a possibly cached list of submission data + _submissions, _total = get_submissions( user_id=perspective_of_id or user.id, course_id=course_id, assignment_id=assignment_id, + limit=limit, + offset=offset, ) - if submissions_ is None: + # If the submissions query returned None, something went wrong + if _submissions is None: return error_response("Bad Request"), 400 # Get submissions through cached function - return success_response({"submissions": submissions_}) + return success_response({ + "submissions": _submissions, + "total": _total, + "user": perspective_of.data + }) @submissions.route("/get/") @@ -94,36 +113,42 @@ def public_submission(commit: str): return success_response({"submission": s.full_data}) -@submissions.route("/regrade/") +@submissions.route("/regrade/") @require_user() @log_endpoint("regrade-request", lambda: "submission regrade request " + request.path) @json_response -def public_regrade_commit(commit=None): +def public_regrade_commit(commit: str): """ This route will get hit whenever someone clicks the regrade button on a processed assignment. It should do some validity checks on the commit and netid, then reset the submission and re-enqueue the submission job. """ - if commit is None: - return error_response("incomplete_request"), 406 # Load current user user: User = current_user() # Find the submission - submission: Submission = ( - Submission.query.join(User) - .filter(Submission.commit == commit, User.netid == user.netid) - .first() - ) + submission: Submission = Submission.query.filter( + Submission.commit == commit + ).first() # Verify Ownership if submission is None: - return error_response("invalid commit hash or netid"), 406 + return error_response("invalid commit hash") + + # Check that the owner matches the user + if submission.owner_id != user.id: + + # If the user is not the owner, then full stop if + assert_course_context(submission) # Check that autograde is enabled for the assignment if not submission.assignment.autograde_enabled: - return error_response('Autograde is disabled for this assignment'), 400 + return error_response('Autograde is disabled for this assignment') + + # Check that the submission is allowed to be accepted + if not submission.accepted: + return error_response('Submission was rejected for being late') # Regrade return regrade_submission(submission) diff --git a/api/anubis/views/public/visuals.py b/api/anubis/views/public/visuals.py index 23f6d042e..e9235b9d9 100644 --- a/api/anubis/views/public/visuals.py +++ b/api/anubis/views/public/visuals.py @@ -33,6 +33,7 @@ def public_visuals_usage(): @visuals.route('/raw-usage') +@cache.cached(timeout=360, unless=is_debug) def public_visuals_raw_usage(): """ Get the raw usage data for generating a react-vis diff --git a/api/anubis/views/public/webhook.py b/api/anubis/views/public/webhook.py index c9ed9dd60..ee394e7d9 100644 --- a/api/anubis/views/public/webhook.py +++ b/api/anubis/views/public/webhook.py @@ -15,13 +15,14 @@ from anubis.utils.data import is_debug from anubis.utils.http.decorators import json_response from anubis.utils.http.https import error_response, success_response +from anubis.utils.lms.assignments import get_assignment_due_date from anubis.utils.lms.webhook import parse_webhook, guess_github_username, check_repo from anubis.utils.lms.submissions import get_submissions from anubis.utils.lms.repos import get_repos from anubis.utils.services.elastic import log_endpoint, esindex from anubis.utils.services.logger import logger from anubis.utils.services.rpc import enqueue_autograde_pipeline -from anubis.utils.services.cache import cache +from anubis.utils.lms.submissions import reject_late_submission, init_submission webhook = Blueprint("public-webhook", __name__, url_prefix="/public/webhook") @@ -176,7 +177,7 @@ def public_webhook(): return success_response({'status': 'already created'}) # Create the related submission models - submission.init_submission_models() + init_submission(submission) # If a user has not given us their github username # the submission will stay in a "dangling" state @@ -209,11 +210,21 @@ def public_webhook(): # if the github username is not found, create a dangling submission if assignment.autograde_enabled: - enqueue_autograde_pipeline(submission.id) + + # Check that the current assignment is still accepting submissions + if not assignment.accept_late and datetime.now() < get_assignment_due_date(user, assignment): + reject_late_submission(submission) + else: - submission.processed = 1 + submission.processed = True + submission.accepted = False submission.state = 'autograde disabled for this assignment' - db.session.commit() + + db.session.commit() + + # If the submission was accepted, then enqueue the job + if submission.accepted and user is not None: + enqueue_autograde_pipeline(submission.id) # Delete cached submissions cache.delete_memoized(get_submissions, user.netid) diff --git a/api/jobs/reaper.py b/api/jobs/reaper.py index 621e1fced..d45e91a68 100644 --- a/api/jobs/reaper.py +++ b/api/jobs/reaper.py @@ -4,13 +4,14 @@ from datetime import datetime, timedelta import requests -from sqlalchemy import func, and_ from anubis.models import db, Submission, Assignment, AssignmentRepo from anubis.utils.data import with_context from anubis.utils.lms.autograde import bulk_autograde +from anubis.utils.lms.submissions import init_submission from anubis.utils.lms.webhook import check_repo, guess_github_username from anubis.utils.services.rpc import enqueue_ide_reap_stale, enqueue_autograde_pipeline +rom anubis.utils.visuals.assignments import get_assignment_sundial def reap_stale_submissions(): @@ -40,7 +41,7 @@ def reap_stale_submissions(): db.session.commit() -def reap_broken_submissions(): +def reap_recent_assignments(): """ Calculate stats for recent submissions @@ -48,13 +49,9 @@ def reap_broken_submissions(): """ from anubis.config import config - recent_assignments = Assignment.query.group_by( - Assignment.course_id - ).having( - and_( - Assignment.release_date == func.max(Assignment.release_date), - Assignment.due_date + config.STATS_REAP_DURATION > datetime.now(), - ) + recent_assignments = Assignment.query.filter( + Assignment.release_date > datetime.now(), + Assignment.due_date > datetime.now() - config.STATS_REAP_DURATION, ).all() print(json.dumps({ @@ -67,7 +64,7 @@ def reap_broken_submissions(): Submission.build == None, ).all(): if submission.build is None: - submission.init_submission_models() + init_submission(submission) enqueue_autograde_pipeline(submission.id) for assignment in recent_assignments: @@ -94,24 +91,24 @@ def reap_broken_repos(): # Do graphql nonsense query = ''' query{ - organization(login:"os3224"){ - repositories(first:100,orderBy:{field:CREATED_AT,direction:DESC}){ - nodes{ - ref(qualifiedName:"master") { - target { - ... on Commit { - history(first: 20) { - edges { node { oid } } + organization(login:"os3224"){ + repositories(first:100,orderBy:{field:CREATED_AT,direction:DESC}){ + nodes{ + ref(qualifiedName:"master") { + target { + ... on Commit { + history(first: 20) { + edges { node { oid } } + } + } + } + } + name + url } } } - } - name - url - } } - } -} ''' url = 'https://api.github.com/graphql' json = {'query': query} @@ -190,7 +187,7 @@ def reap_broken_repos(): ) db.session.add(submission) db.session.commit() - submission.init_submission_models() + init_submission(submission) enqueue_autograde_pipeline(submission.id) r = AssignmentRepo.query.filter(AssignmentRepo.repo_url == repo_url).first() @@ -225,7 +222,7 @@ def reap(): reap_broken_repos() # Reap broken submissions in recent assignments - reap_broken_submissions() + reap_recent_assignments() if __name__ == "__main__": diff --git a/api/jobs/visuals.py b/api/jobs/visuals.py index 12c8863d8..e9b52dc9a 100644 --- a/api/jobs/visuals.py +++ b/api/jobs/visuals.py @@ -1,13 +1,24 @@ -from datetime import datetime +from datetime import datetime, timedelta +from typing import List from anubis.utils.data import with_context +from anubis.models import Assignment from anubis.utils.visuals.usage import get_usage_plot +from anubis.utils.visuals.assignments import get_assignment_sundial @with_context def main(): get_usage_plot() + recent_assignments: List[Assignment] = Assignment.query.filter( + Assignment.release_date > datetime.now(), + Assignment.due_date < datetime.now() - timedelta(weeks=4) + ).all() + + for assignment in recent_assignments: + get_assignment_sundial(assignment.id) + if __name__ == "__main__": print(f"Running visuals job - {datetime.now()}") diff --git a/api/migrations/versions/3ed4da84667e_add_accept_late_to_assignments.py b/api/migrations/versions/3ed4da84667e_add_accept_late_to_assignments.py new file mode 100644 index 000000000..b0f4da4e6 --- /dev/null +++ b/api/migrations/versions/3ed4da84667e_add_accept_late_to_assignments.py @@ -0,0 +1,33 @@ +"""ADD accept_late to assignments + +Revision ID: 3ed4da84667e +Revises: 2324a3537ff3 +Create Date: 2021-05-19 20:16:16.729701 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "3ed4da84667e" +down_revision = "2324a3537ff3" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "assignment", sa.Column("accept_late", sa.Boolean(), nullable=True) + ) + conn = op.get_bind() + with conn.begin(): + conn.execute('update assignment set accept_late = 1;') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("assignment", "accept_late") + # ### end Alembic commands ### diff --git a/api/migrations/versions/716e3e14891d_add_late_exceptions.py b/api/migrations/versions/716e3e14891d_add_late_exceptions.py new file mode 100644 index 000000000..4c48dc567 --- /dev/null +++ b/api/migrations/versions/716e3e14891d_add_late_exceptions.py @@ -0,0 +1,44 @@ +"""ADD late_exceptions + +Revision ID: 716e3e14891d +Revises: a622f56b9050 +Create Date: 2021-05-19 22:20:35.197173 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "716e3e14891d" +down_revision = "a622f56b9050" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "late_exception", + sa.Column("user_id", sa.String(length=128), nullable=False), + sa.Column("assignment_id", sa.String(length=128), nullable=False), + sa.Column("due_date", sa.DateTime(), nullable=False), + sa.Column("created", sa.DateTime(), nullable=True), + sa.Column("last_updated", sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ["assignment_id"], + ["assignment.id"], + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["user.id"], + ), + sa.PrimaryKeyConstraint("user_id", "assignment_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("late_exception") + # ### end Alembic commands ### diff --git a/api/migrations/versions/a622f56b9050_add_accepted_column.py b/api/migrations/versions/a622f56b9050_add_accepted_column.py new file mode 100644 index 000000000..418dbde3f --- /dev/null +++ b/api/migrations/versions/a622f56b9050_add_accepted_column.py @@ -0,0 +1,66 @@ +"""ADD accepted column + +Revision ID: a622f56b9050 +Revises: bf3ae1de1d12 +Create Date: 2021-05-19 20:33:45.552222 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "a622f56b9050" +down_revision = "bf3ae1de1d12" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("name", table_name="assignment") + op.drop_index("pipeline_image", table_name="assignment") + op.alter_column( + "assignment_question", + "pool", + existing_type=mysql.INTEGER(display_width=11), + nullable=False, + ) + op.drop_index( + "ix_assignment_question_sequence", table_name="assignment_question" + ) + op.create_index( + op.f("ix_assignment_question_pool"), + "assignment_question", + ["pool"], + unique=False, + ) + op.add_column( + "submission", sa.Column("accepted", sa.Boolean(), nullable=True) + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("submission", "accepted") + op.drop_index( + op.f("ix_assignment_question_pool"), table_name="assignment_question" + ) + op.create_index( + "ix_assignment_question_sequence", + "assignment_question", + ["pool"], + unique=False, + ) + op.alter_column( + "assignment_question", + "pool", + existing_type=mysql.INTEGER(display_width=11), + nullable=True, + ) + op.create_index( + "pipeline_image", "assignment", ["pipeline_image"], unique=False + ) + op.create_index("name", "assignment", ["name"], unique=False) + # ### end Alembic commands ### diff --git a/api/migrations/versions/bf3ae1de1d12_chg_rename_sequence_to_pool.py b/api/migrations/versions/bf3ae1de1d12_chg_rename_sequence_to_pool.py new file mode 100644 index 000000000..3ba7ca117 --- /dev/null +++ b/api/migrations/versions/bf3ae1de1d12_chg_rename_sequence_to_pool.py @@ -0,0 +1,28 @@ +"""CHG rename sequence to pool + +Revision ID: bf3ae1de1d12 +Revises: 3ed4da84667e +Create Date: 2021-05-19 20:20:32.269570 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "bf3ae1de1d12" +down_revision = "3ed4da84667e" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('assignment_question', 'sequence', new_column_name='pool', type_=sa.Integer()) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('assignment_question', 'pool', new_column_name='sequence', type_=sa.Integer()) + # ### end Alembic commands ### diff --git a/api/requirements.txt b/api/requirements.txt index aeb41cec4..9fcdbaef8 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -25,3 +25,4 @@ sqlalchemy-json pandas matplotlib numpy +click==7.1.2 diff --git a/api/tests/test.sh b/api/tests/test.sh index e63730ccd..11308fd7c 100755 --- a/api/tests/test.sh +++ b/api/tests/test.sh @@ -17,7 +17,7 @@ popd export PYTHONPATH="${TEST_ROOT}:${API_ROOT}" DISABLE_ELK=1 DB_HOST=127.0.0.1 if (( $# == 0 )); then echo 'seeding data...' - python seed.py &>/dev/null + python seed.py 1>/dev/null fi echo 'Running tests...' diff --git a/api/tests/test_assignment_admin.py b/api/tests/test_assignment_admin.py index edb24d324..a992d0f2e 100644 --- a/api/tests/test_assignment_admin.py +++ b/api/tests/test_assignment_admin.py @@ -16,8 +16,8 @@ }, "description": "This is a very long description that encompasses the entire assignment\n", "questions": [ - {"sequence": 1, "questions": [{"q": "What is 3*4?", "a": "12"}, {"q": "What is 3*2", "a": "6"}]}, - {"sequence": 2, "questions": [{"q": "What is sqrt(144)?", "a": "12"}]} + {"pool": 1, "questions": [{"q": "What is 3*4?", "a": "12"}, {"q": "What is 3*2", "a": "6"}]}, + {"pool": 2, "questions": [{"q": "What is sqrt(144)?", "a": "12"}]} ], "tests": ["abc123"], } @@ -30,8 +30,8 @@ def test_assignment_admin(): assignment = superuser.get('/admin/assignments/list')['assignments'][0] assignment_id = assignment['id'] - _assignment = superuser.get(f'/admin/assignments/get/{assignment_id}')['assignment'] - assignment_test_id = _assignment['tests'][0]['id'] + _tests = superuser.get(f'/admin/assignments/get/{assignment_id}')['tests'] + assignment_test_id = _tests[0]['id'] permission_test(f'/admin/assignments/get/{assignment_id}') permission_test(f'/admin/assignments/assignment/{assignment_id}/questions/get/student') diff --git a/api/tests/test_questions_admin.py b/api/tests/test_questions_admin.py index e4add29c1..5218868b7 100644 --- a/api/tests/test_questions_admin.py +++ b/api/tests/test_questions_admin.py @@ -25,13 +25,13 @@ def test_questions_admin(): permission_test(f'/admin/questions/add/{unique_code}') questions = superuser.get(f'/admin/questions/get/{unique_code}')['questions'] print(questions) - question_id = questions['0'][0]['id'] + question_id = questions[0]['id'] permission_test(f'/admin/questions/update/{question_id}', method='post', json={'question': sample_question}) - superuser.get(f'/admin/questions/delete/{questions["0"][0]["id"]}') - professor.get(f'/admin/questions/delete/{questions["0"][1]["id"]}') - ta.get(f'/admin/questions/delete/{questions["0"][2]["id"]}') - student.get(f'/admin/questions/delete/{questions["0"][3]["id"]}', should_fail=True) + superuser.get(f'/admin/questions/delete/{questions[0]["id"]}') + professor.get(f'/admin/questions/delete/{questions[1]["id"]}') + ta.get(f'/admin/questions/delete/{questions[2]["id"]}') + student.get(f'/admin/questions/delete/{questions[3]["id"]}', should_fail=True) permission_test(f'/admin/questions/add/{unique_code}') superuser.get(f'/admin/questions/reset-assignments/{unique_code}') diff --git a/api/tests/test_visuals_admin.py b/api/tests/test_visuals_admin.py index 89e340c8c..1f6b0cef7 100644 --- a/api/tests/test_visuals_admin.py +++ b/api/tests/test_visuals_admin.py @@ -7,4 +7,7 @@ def test_visuals_admin(): student = Session('student') student.get(f'/admin/visuals/assignment/{assignment_id}', should_fail=True) + permission_test(f'/admin/visuals/assignment/{assignment_id}') + permission_test(f'/admin/visuals/sundial/{assignment_id}') + permission_test(f'/admin/visuals/history/{assignment_id}/superuser') diff --git a/k8s/chart/templates/reaper-cron.yml b/k8s/chart/templates/reaper-cron.yml index f4aa32a56..dc5d201ad 100644 --- a/k8s/chart/templates/reaper-cron.yml +++ b/k8s/chart/templates/reaper-cron.yml @@ -29,6 +29,8 @@ spec: value: "/opt/app" - name: "DEBUG" value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }} + - name: "JOB" + value: "1" - name: "DISABLE_ELK" value: "0" - name: "LOGGER_NAME" diff --git a/k8s/chart/templates/rpc.yml b/k8s/chart/templates/rpc.yml index 59e3391b3..ce3af8e80 100644 --- a/k8s/chart/templates/rpc.yml +++ b/k8s/chart/templates/rpc.yml @@ -49,6 +49,8 @@ spec: env: - name: "DEBUG" value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }} + - name: "JOB" + value: "1" - name: "IMAGE_PULL_POLICY" value: {{ .Values.imagePullPolicy }} # sqlalchemy uri @@ -108,6 +110,8 @@ spec: env: - name: "DEBUG" value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }} + - name: "JOB" + value: "1" - name: "IMAGE_PULL_POLICY" value: {{ .Values.imagePullPolicy }} # sqlalchemy uri @@ -166,6 +170,8 @@ spec: env: - name: "DEBUG" value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }} + - name: "JOB" + value: "1" - name: "IMAGE_PULL_POLICY" value: {{ .Values.imagePullPolicy }} # sqlalchemy uri diff --git a/k8s/chart/templates/visuals-cron.yml b/k8s/chart/templates/visuals-cron.yml index a72c2ff04..f11f797fc 100644 --- a/k8s/chart/templates/visuals-cron.yml +++ b/k8s/chart/templates/visuals-cron.yml @@ -29,6 +29,8 @@ spec: value: "/opt/app" - name: "DEBUG" value: {{- if .Values.debug }} "1"{{- else }} "0"{{- end }} + - name: "JOB" + value: "1" - name: "DISABLE_ELK" value: "0" - name: "LOGGER_NAME" diff --git a/k8s/debug/provision.sh b/k8s/debug/provision.sh index e306e22f6..7f59b796a 100755 --- a/k8s/debug/provision.sh +++ b/k8s/debug/provision.sh @@ -45,7 +45,6 @@ fi minikube start \ --feature-gates=TTLAfterFinished=true \ --ports=80:80,443:443 \ - --network-plugin=cni \ --cpus=${CPUS} \ --memory=${MEM} \ --cni=calico diff --git a/theia/ide/admin/cli/anubis/assignment/Dockerfile b/theia/ide/admin/cli/anubis/assignment/Dockerfile index a19979c83..c54839538 100644 --- a/theia/ide/admin/cli/anubis/assignment/Dockerfile +++ b/theia/ide/admin/cli/anubis/assignment/Dockerfile @@ -8,17 +8,15 @@ RUN set -ex; apt update \ && apt autoremove -y \ && find / -name .cache -exec 'rm' '-rf' '{}' ';' \ && rm -rf /var/cache/apt/* \ - && rm -rf /var/lib/apt/lists/* -RUN useradd --no-create-home -u 1001 student \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --no-create-home -u 1001 student \ && mkdir -p /root/anubis/student \ && chmod 700 -R /root \ && chown student:student /root/anubis/student WORKDIR /root/anubis -COPY pipeline.py pipeline.py -COPY utils.py utils.py +COPY pipeline.py utils.py meta.yml /root/anubis/ COPY assignment.py assignment.py -COPY meta.yml meta.yml # COPY student /student CMD python3 pipeline.py diff --git a/theia/ide/admin/cli/anubis/assignment/assignment.py b/theia/ide/admin/cli/anubis/assignment/assignment.py index 44bd847b7..7f09c821f 100644 --- a/theia/ide/admin/cli/anubis/assignment/assignment.py +++ b/theia/ide/admin/cli/anubis/assignment/assignment.py @@ -1,10 +1,12 @@ -from utils import register_test, register_build, exec_as_student -from utils import TestResult, BuildResult, Panic, DEBUG, xv6_run, did_xv6_crash, verify_expected import os +from utils import ( + TestResult, BuildResult, Panic, DEBUG, xv6_run, did_xv6_crash, verify_expected, + register_test, register_build, exec_as_student, search_lines, test_lines +) if DEBUG: - os.environ['GIT_REPO'] = 'https://github.com/juan-punchman/xv6-public.git' + os.environ['GIT_REPO'] = '/student' os.environ['TOKEN'] = 'null' os.environ['COMMIT'] = 'null' os.environ['SUBMISSION_ID'] = 'null' @@ -57,5 +59,3 @@ def test_2(test_result: TestResult): # Test to see if the expected result was found verify_expected(stdout_lines, expected, test_result) - - diff --git a/theia/ide/admin/cli/anubis/assignment/meta.yml b/theia/ide/admin/cli/anubis/assignment/meta.yml index 8d600240f..9ec082359 100644 --- a/theia/ide/admin/cli/anubis/assignment/meta.yml +++ b/theia/ide/admin/cli/anubis/assignment/meta.yml @@ -25,14 +25,14 @@ assignment: # need to manually edit the database. This is by design to prevent # a question assigned to a student from being changed to prevent # confusion. - questions: - - sequence: 1 - questions: - - q: "What is 3*4?" - a: "12" - - q: "What is 3*2" - a: "6" - - sequence: 2 - questions: - - q: "What is sqrt(144)?" - a: "12" + questions: [] + # - pool: 1 + # questions: + # - q: "What is 3*4?" + # a: "12" + # - q: "What is 3*2" + # a: "6" + # - pool: 2 + # questions: + # - q: "What is sqrt(144)?" + # a: "12" diff --git a/theia/ide/admin/cli/anubis/assignment/pipeline.py b/theia/ide/admin/cli/anubis/assignment/pipeline.py index 8d7a0a5a1..d78c47e81 100755 --- a/theia/ide/admin/cli/anubis/assignment/pipeline.py +++ b/theia/ide/admin/cli/anubis/assignment/pipeline.py @@ -13,32 +13,6 @@ root_logger.setLevel(logging.DEBUG) root_logger.addHandler(logging.StreamHandler()) -try: - import assignment -except ImportError: - report_panic('Unable to import assignment', traceback.format_exc()) - exit(0) - -from utils import registered_tests, build_function -from utils import fix_permissions, Panic, DEBUG - -git_creds = os.environ.get('GIT_CRED', default=None) -if git_creds is not None: - del os.environ['GIT_CRED'] - with open(os.environ.get('HOME') + '/.git-credentials', 'w') as f: - f.write(git_creds) - f.close() - with open(os.environ.get('HOME') + '/.gitconfig', 'w') as f: - f.write('[credential]\n') - f.write('\thelper = store\n') - f.close() - -TOKEN = os.environ.get('TOKEN') -COMMIT = os.environ.get('COMMIT') -GIT_REPO = os.environ.get('GIT_REPO') -SUBMISSION_ID = os.environ.get('SUBMISSION_ID') -del os.environ['TOKEN'] - def post(path: str, data: dict, params=None): if params is None: @@ -90,6 +64,33 @@ def report_panic(message: str, traceback: str, ): post('/pipeline/report/panic/{}'.format(SUBMISSION_ID), data) +try: + import assignment +except ImportError: + report_panic('Unable to import assignment', traceback.format_exc()) + exit(0) + +from utils import registered_tests, build_function +from utils import fix_permissions, Panic, DEBUG + +git_creds = os.environ.get('GIT_CRED', default=None) +if git_creds is not None: + del os.environ['GIT_CRED'] + with open(os.environ.get('HOME') + '/.git-credentials', 'w') as f: + f.write(git_creds) + f.close() + with open(os.environ.get('HOME') + '/.gitconfig', 'w') as f: + f.write('[credential]\n') + f.write('\thelper = store\n') + f.close() + +TOKEN = os.environ.get('TOKEN') +COMMIT = os.environ.get('COMMIT') +GIT_REPO = os.environ.get('GIT_REPO') +SUBMISSION_ID = os.environ.get('SUBMISSION_ID') +del os.environ['TOKEN'] + + def report_state(state: str, params=None): """ Report a state update for the current submission diff --git a/theia/ide/admin/cli/anubis/assignment/utils.py b/theia/ide/admin/cli/anubis/assignment/utils.py index 27c2b2e59..131bff20b 100644 --- a/theia/ide/admin/cli/anubis/assignment/utils.py +++ b/theia/ide/admin/cli/anubis/assignment/utils.py @@ -156,6 +156,33 @@ def trim(stdout: str) -> typing.List[str]: return stdout_lines +def search_lines(stdout_lines: typing.List[str], expected_lines: typing.List[str], case_sensitive: bool = True): + if not case_sensitive: + stdout_lines = list(map(lambda x: x.lower(), stdout_lines)) + found = [] + for line in expected_lines: + l = line.strip() + if not case_sensitive: + l = l.lower() + for _aindex, _aline in enumerate(stdout_lines): + if l in _aline: + found.append(_aindex) + break + else: + found.append(-1) + if -1 in found: + return False + return list(sorted(found)) == found + + +def test_lines(stdout_lines: typing.List[str], expected_lines: typing.List[str], case_sensitive: bool = True): + if case_sensitive: + return len(stdout_lines) == len(expected_lines) \ + and all(_a.strip() == _b.strip() for _a, _b in zip(stdout_lines, expected_lines)) + return len(stdout_lines) == len(expected_lines) \ + and all(_a.lower().strip() == _b.lower().strip() for _a, _b in zip(stdout_lines, expected_lines)) + + def verify_expected( stdout_lines: typing.List[str], expected_lines: typing.List[str], @@ -174,34 +201,8 @@ def verify_expected( :return: """ - def search_lines(a: typing.List[str], b: typing.List[str]): - if not case_sensitive: - a = list(map(lambda x: x.lower(), a)) - found = [] - for line in b: - l = line.strip() - if not case_sensitive: - l = l.lower() - for _aindex, _aline in enumerate(a): - if l in _aline: - found.append(_aindex) - break - else: - found.append(-1) - if -1 in found: - return False - - return list(sorted(found)) == found - - def test_lines(a: typing.List[str], b: typing.List[str]): - if case_sensitive: - return len(a) == len(b) \ - and all(_a.strip() == _b.strip() for _a, _b in zip(a, b)) - return len(a) == len(b) \ - and all(_a.lower().strip() == _b.lower().strip() for _a, _b in zip(a, b)) - compare_func = search_lines if search else test_lines - if not compare_func(stdout_lines, expected_lines): + if not compare_func(stdout_lines, expected_lines, case_sensitive=case_sensitive): test_result.stdout += 'your lines:\n' + '\n'.join(stdout_lines) + '\n\n' \ + 'we expected:\n' + '\n'.join(expected_lines) test_result.message = 'Did not receive expected output' diff --git a/theia/ide/admin/cli/anubis/cli.py b/theia/ide/admin/cli/anubis/cli.py index 8f6deeac0..dc190d54b 100644 --- a/theia/ide/admin/cli/anubis/cli.py +++ b/theia/ide/admin/cli/anubis/cli.py @@ -184,6 +184,9 @@ def whoami(): @questions.command() def assign(): + if not os.path.exists('meta.yml'): + click.echo('no meta.yml found!') + return 1 assignment_meta = yaml.safe_load(open('meta.yml').read()) unique_code = assignment_meta['assignment']['unique_code'] get_json('/private/questions/assign/{}'.format(unique_code)) @@ -191,6 +194,9 @@ def assign(): @assignment.command() def sync(): + if not os.path.exists('meta.yml'): + click.echo('no meta.yml found!') + return 1 assignment_meta = yaml.safe_load(open('meta.yml').read()) # click.echo(json.dumps(assignment_meta, indent=2)) import assignment @@ -265,6 +271,9 @@ def stats(assignment, netids): @click.argument('path', type=click.Path(exists=True), default='.') @click.option('--push/-p', default=False) def build(path, push): + if not os.path.exists('meta.yml'): + click.echo('no meta.yml found!') + return 1 assignment_meta = yaml.safe_load(open(os.path.join(path, 'meta.yml')).read()) # Build assignment image diff --git a/web/public/index.html b/web/public/index.html index e06995ce4..0b262f96d 100644 --- a/web/public/index.html +++ b/web/public/index.html @@ -36,7 +36,7 @@ -> Use linux --> - +
diff --git a/web/public/robots.txt b/web/public/robots.txt index 8ee00fef1..b734e37de 100644 --- a/web/public/robots.txt +++ b/web/public/robots.txt @@ -1,5 +1,10 @@ -# https://www.robotstxt.org/robotstxt.html +# Did you think you would find something here? + User-agent: * Allow: /about +Allow: /blog +Allow: /visuals Allow: /favicon.ico -Disallow: * +Allow: /api/public/static/ +Allow: /api/public/visuals/ +Disallow: /api diff --git a/web/src/Components/Admin/Assignment/AssignmentCard.jsx b/web/src/Components/Admin/Assignment/AssignmentCard.jsx index 779ae0cf9..bc86532f8 100644 --- a/web/src/Components/Admin/Assignment/AssignmentCard.jsx +++ b/web/src/Components/Admin/Assignment/AssignmentCard.jsx @@ -20,10 +20,11 @@ import IconButton from '@material-ui/core/IconButton'; import Typography from '@material-ui/core/Typography'; import Tooltip from '@material-ui/core/Tooltip'; - import SaveIcon from '@material-ui/icons/Save'; import EditIcon from '@material-ui/icons/Edit'; import RefreshIcon from '@material-ui/icons/Refresh'; +import VisibilityIcon from '@material-ui/icons/Visibility'; +import AccessTimeIcon from '@material-ui/icons/AccessTime'; import standardErrorHandler from '../../../Utils/standardErrorHandler'; import standardStatusHandler from '../../../Utils/standardStatusHandler'; @@ -220,12 +221,35 @@ export default function AssignmentCard({assignment, editableFields, updateField, size={'small'} color={'primary'} variant={'contained'} + className={classes.button} component={Link} to={`/admin/assignment/tests/${assignment.id}`} startIcon={} > Edit Tests + + diff --git a/web/src/Components/Admin/Assignment/AssignmentReposTable.jsx b/web/src/Components/Admin/Assignment/AssignmentReposTable.jsx new file mode 100644 index 000000000..9f716ba8d --- /dev/null +++ b/web/src/Components/Admin/Assignment/AssignmentReposTable.jsx @@ -0,0 +1,31 @@ +import React from 'react'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import {DataGrid} from '@material-ui/data-grid'; +import Paper from '@material-ui/core/Paper'; +import Typography from '@material-ui/core/Typography'; + +const useStyles = makeStyles((theme) => ({ + paper: { + padding: theme.spacing(1), + height: 700, + }, +})); + +export default function AssignmentReposTable({repos}) { + const classes = useStyles(); + + return ( + + ( + + repo + + )}, + ]} rows={repos}/> + + ); +} diff --git a/web/src/Components/Admin/Assignment/LateExceptionAddCard.jsx b/web/src/Components/Admin/Assignment/LateExceptionAddCard.jsx new file mode 100644 index 000000000..dfcd0ec12 --- /dev/null +++ b/web/src/Components/Admin/Assignment/LateExceptionAddCard.jsx @@ -0,0 +1,106 @@ +import React, {useState} from 'react'; +import {useSnackbar} from 'notistack'; +import axios from 'axios'; + +import {KeyboardDatePicker, KeyboardTimePicker, MuiPickersUtilsProvider} from '@material-ui/pickers'; + +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; +import Card from '@material-ui/core/Card'; +import CardContent from '@material-ui/core/CardContent'; +import DateFnsUtils from '@date-io/date-fns'; +import Autocomplete from '@material-ui/lab/Autocomplete'; +import TextField from '@material-ui/core/TextField'; + +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import standardErrorHandler from '../../../Utils/standardErrorHandler'; +import CardActionArea from '@material-ui/core/CardActionArea'; +import {CardActions} from '@material-ui/core'; +import Button from '@material-ui/core/Button'; +import SaveIcon from '@material-ui/icons/Save'; +import {nonStupidDatetimeFormat} from '../../../Utils/datetime'; +import Typography from '@material-ui/core/Typography'; + +const useStyles = makeStyles((theme) => ({ + datePicker: { + marginRight: theme.spacing(2), + }, + button: { + marginRight: theme.spacing(1), + }, +})); + +export default function LateExceptionAddCard({assignment, setReset}) { + const classes = useStyles(); + const {enqueueSnackbar} = useSnackbar(); + const [date, setDate] = useState(new Date(assignment.due_date)); + const [students, setStudents] = useState([]); + const [selected, setSelected] = useState(null); + + React.useEffect(() => { + axios.get(`/api/admin/students/list`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.students) { + setStudents(data.students); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }, []); + + const save = () => { + if (selected === null) { + enqueueSnackbar('No student selected', {variant: 'error'}); + return; + } + console.log(date); + axios.post(`/api/admin/late-exceptions/update`, { + assignment_id: assignment.id, + user_id: selected.id, + due_date: nonStupidDatetimeFormat(date), + }).then((response) => { + standardStatusHandler(response, enqueueSnackbar); + setReset((prev) => ++prev); + }).catch(standardErrorHandler(enqueueSnackbar)); + }; + + return ( + + + option.netid} + renderInput={(params) => } + options={students} + onChange={(_, v) => setSelected(v)} + /> + + setDate(v)} + /> + setDate(v)} + /> + + + + + + + ); +} diff --git a/web/src/Components/Admin/Assignment/RepoCommandDialog.jsx b/web/src/Components/Admin/Assignment/RepoCommandDialog.jsx new file mode 100644 index 000000000..b175af29a --- /dev/null +++ b/web/src/Components/Admin/Assignment/RepoCommandDialog.jsx @@ -0,0 +1,58 @@ +import React from 'react'; + +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Button from '@material-ui/core/Button'; +import Dialog from '@material-ui/core/Dialog'; +import DialogActions from '@material-ui/core/DialogActions'; +import DialogContent from '@material-ui/core/DialogContent'; +import DialogContentText from '@material-ui/core/DialogContentText'; +import DialogTitle from '@material-ui/core/DialogTitle'; +import FormControlLabel from '@material-ui/core/FormControlLabel'; +import Switch from '@material-ui/core/Switch'; + + +const useStyles = makeStyles((theme) => ({ + root: { + flex: 1, + }, +})); + +export default function RepoCommandDialog({repos = [], assignment = {}}) { + const [open, setOpen] = React.useState(false); + const [http, setHttp] = React.useState(false); + + const handleClickOpen = () => setOpen(true); + const handleClose = () => setOpen(false); + + return ( +
+ + + + setHttp(!http)} + name="http" + color="primary" + /> + } + label={http ? 'http' : 'ssh'} + /> +

+ mkdir -p '{assignment?.name}'
+ cd '{assignment?.name}'
+ {repos.map(({url, ssh, netid}) => ( +

+ git clone {http ? url : ssh} {netid} +
+ ))} +

+ + +
+ ); +} diff --git a/web/src/Components/Admin/Users/CourseCard.jsx b/web/src/Components/Admin/Users/CourseCard.jsx index dd6dc287f..b81c0106e 100644 --- a/web/src/Components/Admin/Users/CourseCard.jsx +++ b/web/src/Components/Admin/Users/CourseCard.jsx @@ -18,7 +18,7 @@ const useStyles = makeStyles((theme) => ({ })); -export default function CourseCard({student, course}) { +export default function CourseCard({user, course}) { const classes = useStyles(); return ( @@ -59,7 +59,7 @@ export default function CourseCard({student, course}) { color="primary" size="small" component={Link} - to={`/courses/assignments/submissions?courseId=${course.id}&userId=${student.id}`} + to={`/courses/assignments/submissions?courseId=${course.id}&userId=${user.id}`} > View Submissions diff --git a/web/src/Components/Admin/Users/UserCard.jsx b/web/src/Components/Admin/Users/UserCard.jsx index 1233e5eeb..230eaee65 100644 --- a/web/src/Components/Admin/Users/UserCard.jsx +++ b/web/src/Components/Admin/Users/UserCard.jsx @@ -61,7 +61,7 @@ export default function UserCard({user, setUser}) { {/* netid */} - Netid: {user.student.netid} + Netid: {user.netid} @@ -70,10 +70,10 @@ export default function UserCard({user, setUser}) { { setUser((state) => { - state.student.name = e.target.value; + state.name = e.target.value; return state; }); incEdits(); @@ -86,10 +86,10 @@ export default function UserCard({user, setUser}) { { setUser((state) => { - state.github_username.name = e.target.value; + state.github_username = e.target.value; return state; }); incEdits(); @@ -97,45 +97,19 @@ export default function UserCard({user, setUser}) { /> - {/* Is Admin */} - - { - axios.get(`/api/admin/students/toggle-admin/${user.student.id}`).then((response) => { - const data = standardStatusHandler(response, enqueueSnackbar); - if (data) { - setUser((state) => { - state.student.is_admin = !state.student.is_admin; - return state; - }); - incEdits(); - } - }).catch(standardErrorHandler(enqueueSnackbar)); - }} - /> - } - label="Is Admin" - labelPlacement="end" - /> - - {/* Is Superuser */} { axios.get(`/api/admin/students/toggle-superuser/${user.student.id}`).then((response) => { const data = standardStatusHandler(response, enqueueSnackbar); if (data) { setUser((state) => { - state.student.is_superuser = !state.student.is_superuser; + state.is_superuser = !state.is_superuser; return state; }); incEdits(); @@ -157,7 +131,7 @@ export default function UserCard({user, setUser}) { color="primary" size="small" startIcon={} - onClick={() => saveUser(student, enqueueSnackbar)} + onClick={() => saveUser(user, enqueueSnackbar)} > Save diff --git a/web/src/Components/Admin/Visuals/AssignmentSundialPaper.jsx b/web/src/Components/Admin/Visuals/AssignmentSundialPaper.jsx new file mode 100644 index 000000000..23aadfa77 --- /dev/null +++ b/web/src/Components/Admin/Visuals/AssignmentSundialPaper.jsx @@ -0,0 +1,40 @@ +import React, {useState} from 'react'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; +import {useSnackbar} from 'notistack'; +import axios from 'axios'; +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import standardErrorHandler from '../../../Utils/standardErrorHandler'; +import useQuery from '../../../hooks/useQuery'; +import Paper from '@material-ui/core/Paper'; +import AssignmentSundial from './Graphs/AssignmentSundial'; + +const useStyles = makeStyles((theme) => ({ + root: { + flex: 1, + }, +})); + +export default function AssignmentSundialPaper() { + const classes = useStyles(); + const query = useQuery(); + const {enqueueSnackbar} = useSnackbar(); + const [sundialData, setSundialData] = useState([]); + + const assignmentId = query.get('assignmentId'); + + React.useEffect(() => { + axios.get(`/api/admin/visuals/sundial/${assignmentId}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.sundial) { + setSundialData(data?.sundial); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }, []); + + return ( + + + + ); +} diff --git a/web/src/Components/Admin/Visuals/AutogradeVisuals.jsx b/web/src/Components/Admin/Visuals/AutogradeVisuals.jsx index 5300d6972..cf08c86cb 100644 --- a/web/src/Components/Admin/Visuals/AutogradeVisuals.jsx +++ b/web/src/Components/Admin/Visuals/AutogradeVisuals.jsx @@ -6,6 +6,8 @@ import standardErrorHandler from '../../../Utils/standardErrorHandler'; import standardStatusHandler from '../../../Utils/standardStatusHandler'; import AssignmentTestsPaper from './AssignmentTestsPaper'; +import AssignmentSundialPaper from './AssignmentSundialPaper'; +import Grid from '@material-ui/core/Grid'; export default function AutogradeVisuals({assignmentId}) { @@ -22,15 +24,25 @@ export default function AutogradeVisuals({assignmentId}) { }, []); return ( -
- {assignmentData.map(({title, pass_time_scatter, pass_count_radial}) => ( - - ))} -
+ + + + + {assignmentData.length !== 0 ? ( + + + {assignmentData.map(({title, pass_time_scatter, pass_count_radial}) => ( + + + + ))} + + + ) : null} + ); } diff --git a/web/src/Components/Admin/Visuals/Graphs/AssignmentSundial.jsx b/web/src/Components/Admin/Visuals/Graphs/AssignmentSundial.jsx new file mode 100644 index 000000000..7ed312b09 --- /dev/null +++ b/web/src/Components/Admin/Visuals/Graphs/AssignmentSundial.jsx @@ -0,0 +1,135 @@ +import React, {useEffect, useState} from 'react'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import {useSnackbar} from 'notistack'; + +import {LabelSeries, Sunburst} from 'react-vis'; +import {EXTENDED_DISCRETE_COLOR_RANGE} from 'react-vis/es/theme'; +import Typography from '@material-ui/core/Typography'; + +const LABEL_STYLE = { + fontSize: '12px', + color: '#fff', + textAnchor: 'middle', +}; + +const useStyles = makeStyles((theme) => ({ + title: { + fontSize: 18, + paddingLeft: theme.spacing(1), + paddingTop: theme.spacing(1), + }, + typography: { + padding: theme.spacing(1), + }, +})); + +/** + * Recursively work backwards from highlighted node to find path of valud nodes + * @param {Object} node - the current node being considered + * @returns {Array} an array of strings describing the key route to the current node + */ +function getKeyPath(node) { + if (!node.parent) { + return ['root']; + } + + return [(node.data && node.data.name) || node.name].concat( + getKeyPath(node.parent), + ); +} + +/** + * Recursively modify data depending on whether or not each cell has been selected by the hover/highlight + * @param {Object} data - the current node being considered + * @param {Object|Boolean} keyPath - a map of keys that are in the highlight path + * if this is false then all nodes are marked as selected + * @returns {Object} Updated tree structure + */ +function updateData(data, keyPath) { + if (data.children) { + data.children.map((child) => updateData(child, keyPath)); + } + // add a fill to all the uncolored cells + if (!data.hex) { + data.style = { + fill: EXTENDED_DISCRETE_COLOR_RANGE[5], + }; + } + data.style = { + ...data.style, + fillOpacity: keyPath && !keyPath[data.name] ? 0.2 : 1, + }; + + return data; +} + +export default function AssignmentSundial({sundialData}) { + const classes = useStyles(); + const {enqueueSnackbar} = useSnackbar(); + const [pathValue, setPathValue] = useState(false); + const [finalValue, setFinalValue] = useState('Anubis'); + const [data, setData] = useState({}); + const [decoratedData, setDecoratedData] = useState({}); + const [clicked, setClicked] = useState(false); + + useEffect(() => { + setDecoratedData(updateData(sundialData, false)); + setData(updateData(sundialData, false)); + }, [sundialData]); + + return ( +
+ + Anubis Autograde Sundial + + { + if (clicked) { + return; + } + const path = getKeyPath(node).reverse(); + const pathAsMap = path.reduce((res, row) => { + res[row] = true; + return res; + }, {}); + setData(updateData(decoratedData, pathAsMap)); + setPathValue(path.join(' > ')); + setFinalValue(path[path.length - 1]); + }} + onValueMouseOut={() => + clicked ? + () => { + } : (function() { + setPathValue(false); + setFinalValue(false); + setData(updateData(decoratedData, false)); + })() + } + onValueClick={() => setClicked(!clicked)} + style={{ + stroke: '#ddd', + strokeOpacity: 0.3, + strokeWidth: '0.5', + }} + colorType="literal" + getSize={(d) => d.value} + getColor={(d) => d.hex} + data={data} + height={600} + width={650} + > + {finalValue && ( + + )} + + + {pathValue} + +
+ ); +} diff --git a/web/src/Components/Admin/Visuals/Graphs/StudentHistory.jsx b/web/src/Components/Admin/Visuals/Graphs/StudentHistory.jsx new file mode 100644 index 000000000..5081600c9 --- /dev/null +++ b/web/src/Components/Admin/Visuals/Graphs/StudentHistory.jsx @@ -0,0 +1,160 @@ +import React, {useEffect, useState} from 'react'; +import Paper from '@material-ui/core/Paper'; +import Slider from '@material-ui/core/Slider'; +import Typography from '@material-ui/core/Typography'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; + +import { + DiscreteColorLegend, + FlexibleWidthXYPlot, + Hint, + HorizontalGridLines, + LineMarkSeries, + VerticalGridLines, + XAxis, + YAxis, +} from 'react-vis'; + +const useStyles = makeStyles((theme) => ({ + legend: { + color: theme.palette.text.primary, + }, + typography: { + paddingLeft: theme.spacing(1), + }, + slider: { + marginLeft: theme.spacing(1), + marginRight: theme.spacing(1), + }, + button: { + margin: theme.spacing(1), + }, +})); + +export default function StudentHistory({testResults: rawTestResults, buildResults: rawBuildResults}) { + const classes = useStyles(); + const [hint, setHint] = useState(null); + const [width, setWidth] = useState(0); + const [testResults, setTestResult] = useState([]); + const [buildResults, setBuildResult] = useState([]); + // const [lastDrawLocation, setLastDrawLocation] = useState(null); + + useEffect(() => { + setTestResult(rawTestResults.map(({x, ...y}) => ({x: new Date(x), ...y}))); + setBuildResult(rawBuildResults.map(({x, ...y}) => ({x: new Date(x), ...y}))); + setWidth(rawBuildResults.length); + }, [rawBuildResults, rawTestResults]); + + const updateValues = (width) => { + setTestResult(rawTestResults.slice(0, width).map(({x, ...y}) => ({x: new Date(x), ...y}))); + setBuildResult(rawBuildResults.slice(0, width).map(({x, ...y}) => ({x: new Date(x), ...y}))); + setWidth(width); + }; + + return ( + + + + + + + + + Items to Show + + updateValues(value)} + min={5} + max={rawTestResults.length} + valueLabelDisplay="auto" + /> + {/* setLastDrawLocation(null)}*/} + {/* >*/} + {/* Reset Zoom*/} + {/* */} + + + + + + + + + + {/* Tests */} + setHint(value)} + onValueMouseOut={() => setHint(null)} + style={{strokeWidth: '2px'}} + lineStyle={{stroke: 'blue'}} + markStyle={{stroke: 'blue'}} + data={testResults} + /> + + {/* Builds */} + setHint(value)} + onValueMouseOut={() => setHint(null)} + style={{strokeWidth: '2px'}} + lineStyle={{stroke: 'green'}} + markStyle={{stroke: 'green'}} + data={buildResults} + /> + + {/* */} + + {/* setLastDrawLocation(area)}*/} + {/* onDrag={(area) => {*/} + {/* setLastDrawLocation({*/} + {/* bottom: lastDrawLocation.bottom + (area.top - area.bottom),*/} + {/* left: lastDrawLocation.left - (area.right - area.left),*/} + {/* right: lastDrawLocation.right - (area.right - area.left),*/} + {/* top: lastDrawLocation.top + (area.top - area.bottom),*/} + {/* });*/} + {/* }}*/} + {/* />*/} + + {hint ? : null} + + + ); +} diff --git a/web/src/Components/Admin/Visuals/StudentAssignmentHistory.jsx b/web/src/Components/Admin/Visuals/StudentAssignmentHistory.jsx new file mode 100644 index 000000000..c5b991511 --- /dev/null +++ b/web/src/Components/Admin/Visuals/StudentAssignmentHistory.jsx @@ -0,0 +1,38 @@ +import React, {useState} from 'react'; +import {useSnackbar} from 'notistack'; +import axios from 'axios'; +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import standardErrorHandler from '../../../Utils/standardErrorHandler'; +import useQuery from '../../../hooks/useQuery'; + +import StudentHistory from './Graphs/StudentHistory'; + + +export default function StudentAssignmentHistory() { + const query = useQuery(); + const {enqueueSnackbar} = useSnackbar(); + const [testResults, setTestResults] = useState([]); + const [buildResults, setBuildResults] = useState([]); + + const assignmentId = query.get('assignmentId'); + const netid = query.get('netid'); + + React.useEffect(() => { + axios.get(`/api/admin/visuals/history/${assignmentId}/${netid}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.submissions?.test_results) { + setTestResults(data?.submissions.test_results); + } + if (data?.submissions?.build_results) { + setBuildResults(data?.submissions.build_results); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }, []); + + return ( + + ); +} diff --git a/web/src/Components/Header.jsx b/web/src/Components/Header.jsx index 7f9e374df..9cd9bcad6 100644 --- a/web/src/Components/Header.jsx +++ b/web/src/Components/Header.jsx @@ -1,4 +1,4 @@ -import React, {useState} from 'react'; +import React, {useEffect, useState} from 'react'; import Cookies from 'universal-cookie'; import {useSnackbar} from 'notistack'; import clsx from 'clsx'; @@ -26,6 +26,17 @@ export default function Header({classes, open, onDrawerToggle, user}) { return null; })()); + useEffect(() => { + if ((user?.admin_for?.length ?? 0) > 0) { + try { + JSON.parse(atob(cookie.get('course'))); + } catch (_) { + cookie.set('course', btoa(JSON.stringify(user.admin_for[0])), {path: '/'}); + setCourse(user.admin_for[0]); + } + } + }, [user]); + return ( { - setExpanded(!expanded); - }; - - return ( @@ -83,36 +74,26 @@ export default function NotFound() { title={'memes'} /> - - + + - - - setQuote(getQuote())} - className={classes.iconleft} - > - - - - - + + + + setQuote(getQuote())} + className={classes.iconleft} + > + + + - - - - {quote} - - - + + + {quote} + + diff --git a/web/src/Components/Public/Assignments/AssignmentCard.jsx b/web/src/Components/Public/Assignments/AssignmentCard.jsx index 4d5f7b200..019dcc9f9 100644 --- a/web/src/Components/Public/Assignments/AssignmentCard.jsx +++ b/web/src/Components/Public/Assignments/AssignmentCard.jsx @@ -14,6 +14,8 @@ import GitHubIcon from '@material-ui/icons/GitHub'; import red from '@material-ui/core/colors/red'; import green from '@material-ui/core/colors/green'; import CodeOutlinedIcon from '@material-ui/icons/CodeOutlined'; +import ExitToAppIcon from '@material-ui/icons/ExitToApp'; +import {nonStupidDatetimeFormat} from '../../../Utils/datetime'; const useStyles = makeStyles((theme) => ({ root: { @@ -80,8 +82,11 @@ export default function AssignmentCard({assignment, setSelectedTheia}) { course: {course_code}, has_submission, github_classroom_link, - ide_active, + ide_enabled, has_repo, + repo_url, + past_due, + accept_late, } = assignment; const [timeLeft] = useState(remainingTime(due_date)); @@ -116,7 +121,7 @@ export default function AssignmentCard({assignment, setSelectedTheia}) {
-

{` Due: ${(new Date(due_date)).toLocaleDateString()}`}

+

Due: {nonStupidDatetimeFormat(new Date(due_date))}

@@ -142,7 +147,7 @@ export default function AssignmentCard({assignment, setSelectedTheia}) { color={'primary'} className={classes.button} startIcon={} - disabled={!ide_active || !has_repo} + disabled={!ide_enabled || !has_repo || past_due} onClick={() => setSelectedTheia(assignment)} > Anubis Cloud IDE @@ -151,14 +156,13 @@ export default function AssignmentCard({assignment, setSelectedTheia}) { size={'small'} variant={'contained'} color={'primary'} - startIcon={} + startIcon={has_repo ? : } className={classes.button} - disabled={!githubLinkEnabled || has_repo} component={'a'} - href={github_classroom_link} + href={has_repo ? repo_url : github_classroom_link} target={'_blank'} > - Create repo + {has_repo ? 'Go to repo' : 'Create repo'} diff --git a/web/src/Components/Public/Questions/QuestionsCard.jsx b/web/src/Components/Public/Questions/QuestionsCard.jsx index b96c2b282..6a12d67b9 100644 --- a/web/src/Components/Public/Questions/QuestionsCard.jsx +++ b/web/src/Components/Public/Questions/QuestionsCard.jsx @@ -76,11 +76,11 @@ export default function QuestionsCard({questions}) {
- {questions.sort(({question: q1}, {question: q2}) => q1.sequence - q2.sequence).map(({id, question}, index) => ( - + {questions.sort(({question: q1}, {question: q2}) => q1.pool - q2.pool).map(({id, question}, index) => ( + }> - Question {question.sequence} + Question {question.pool} diff --git a/web/src/Components/Public/Submissions/SubmissionsTable.jsx b/web/src/Components/Public/Submissions/SubmissionsTable.jsx deleted file mode 100644 index 4df86eaee..000000000 --- a/web/src/Components/Public/Submissions/SubmissionsTable.jsx +++ /dev/null @@ -1,147 +0,0 @@ -import React from 'react'; -import TableContainer from '@material-ui/core/TableContainer'; -import Paper from '@material-ui/core/Paper'; -import Table from '@material-ui/core/Table'; -import TableHead from '@material-ui/core/TableHead'; -import TableRow from '@material-ui/core/TableRow'; -import TableCell from '@material-ui/core/TableCell'; -import TableBody from '@material-ui/core/TableBody'; -import {Link} from 'react-router-dom'; -import CheckCircleIcon from '@material-ui/icons/CheckCircle'; -import green from '@material-ui/core/colors/green'; -import CancelIcon from '@material-ui/icons/Cancel'; -import red from '@material-ui/core/colors/red'; -import TableFooter from '@material-ui/core/TableFooter'; -import TablePagination from '@material-ui/core/TablePagination'; -import TablePaginationActions from '@material-ui/core/TablePagination/TablePaginationActions'; -import {makeStyles} from '@material-ui/core/styles'; - -const useStyles = makeStyles({ - root: { - flexGrow: 1, - }, - table: { - minWidth: 500, - }, - headerText: { - fontWeight: 600, - }, - commitHashContainer: { - - width: 200, - overflow: 'hidden', - }, -}); - -export function SubmissionsTable({rows}) { - const classes = useStyles(); - const [page, setPage] = React.useState(0); - const [rowsPerPage, setRowsPerPage] = React.useState(10); - - const handleChangePage = (event, newPage) => { - setPage(newPage); - }; - - const handleChangeRowsPerPage = (event) => { - setRowsPerPage(parseInt(event.target.value, 10)); - setPage(0); - }; - - const emptyRows = rowsPerPage - Math.min(rowsPerPage, rows.length - page * rowsPerPage); - - return ( - - - - - - Assignment Name - - - Commit Hash - - - Tests Passed - - - Processed - - - On Time - - - Date - - - Time - - - - - - {(rowsPerPage > 0 ? - rows.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) : - rows - ).map((row) => ( - - - {row.assignmentName} - - - {row.commitHash.substring(0, 10)} - - - {row.testsPassed}/{row.totalTests} - - - {row.processed ? : - } - - - {row.timeStamp <= row.assignmentDue ? : - } - - - {row.timeSubmitted} - - - {row.dateSubmitted} - - - ))} - - {emptyRows > 0 && ( - - - - )} - - - - - - -
- - ); -} diff --git a/web/src/Main.jsx b/web/src/Main.jsx index 7138bc969..bbbe41481 100644 --- a/web/src/Main.jsx +++ b/web/src/Main.jsx @@ -1,16 +1,17 @@ import React from 'react'; import {Redirect, Route, Switch} from 'react-router-dom'; -import {admin_nav, footer_nav, not_shown_nav, public_nav} from './navconfig'; +import {footer_nav, not_shown_nav, public_nav, admin_nav} from './navconfig'; +import NotFound from './Components/NotFound'; export default function Main({user}) { return ( - {public_nav.map(({children}) => children.map(({path, Page, exact=true}) => ( + {public_nav.map(({children}) => children.map(({path, Page, exact = true}) => ( )))} - {footer_nav.map(({path, Page, exact=true}) => ( + {footer_nav.map(({path, Page, exact = true}) => ( @@ -31,7 +32,7 @@ export default function Main({user}) { - 404 not found + ); diff --git a/web/src/Pages/Admin/Assignment/Assignment.jsx b/web/src/Pages/Admin/Assignment/Assignment.jsx new file mode 100644 index 000000000..0580dbb93 --- /dev/null +++ b/web/src/Pages/Admin/Assignment/Assignment.jsx @@ -0,0 +1,121 @@ +import React, {useState} from 'react'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; +import {useSnackbar} from 'notistack'; +import Typography from '@material-ui/core/Typography'; +import ManagementIDEDialog from '../../../Components/Admin/IDE/ManagementIDEDialog'; +import AssignmentCard from '../../../Components/Admin/Assignment/AssignmentCard'; +import axios from 'axios'; +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import {nonStupidDatetimeFormat} from '../../../Utils/datetime'; +import useQuery from '../../../hooks/useQuery'; +import {useParams} from 'react-router-dom'; +import {CircularProgress} from '@material-ui/core'; + +const useStyles = makeStyles((theme) => ({ + root: { + flex: 1, + }, +})); + +const editableFields = [ + {field: 'name', label: 'Assignment Name'}, + {field: 'github_classroom_url', label: 'Github Classroom URL'}, + {field: 'theia_image', label: 'Theia Image'}, + {field: 'theia_options', label: 'Theia Options', type: 'json'}, + {field: 'pipeline_image', label: 'Pipeline Image', disabled: true}, + {field: 'unique_code', label: 'Unique Code', disabled: true}, + {field: 'hidden', label: 'Hidden', type: 'boolean'}, + {field: 'accept_late', label: 'Accept Late Submissions', type: 'boolean'}, + {field: 'ide_enabled', label: 'Theia Enabled', type: 'boolean'}, + {field: 'autograde_enabled', label: 'Autograde Enabled', type: 'boolean'}, + {field: 'release_date', label: 'Release Date', type: 'datetime'}, + {field: 'due_date', label: 'Due Date', type: 'datetime'}, + {field: 'grace_date', label: 'Grace Date', type: 'datetime'}, +]; + +export default function Assignment() { + const classes = useStyles(); + const {enqueueSnackbar} = useSnackbar(); + const [assignment, setAssignment] = useState(null); + const [reset, setReset] = useState(0); + const {assignmentId} = useParams(); + + React.useEffect(() => { + axios.get(`/api/admin/assignments/get/${assignmentId}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.assignment) { + for (const field of editableFields) { + if (field.type === 'datetime') { + console.log(field.field); + data.assignment[field.field] = new Date(data.assignment[field.field].replace(/-/g, '/')); + } + } + setAssignment(data.assignment); + } + }).catch((error) => enqueueSnackbar(error.toString(), {variant: 'error'})); + }, [reset]); + + const updateField = (id, field, toggle = false, datetime = false, json = false) => (e) => { + if (!e) { + return; + } + + if (toggle) { + assignment[field] = !assignment[field]; + } else if (datetime) { + assignment[field] = e; + } else if (json) { + assignment[field] = e; + } else { + assignment[field] = e.target.value.toString(); + } + + setAssignment({...assignment}); + }; + + const saveAssignment = (id) => () => { + if (assignment.id === id) { + const conv_assignment = { + ...assignment, + release_date: nonStupidDatetimeFormat(assignment.release_date), + due_date: nonStupidDatetimeFormat(assignment.due_date), + grace_date: nonStupidDatetimeFormat(assignment.grace_date), + }; + axios.post(`/api/admin/assignments/save`, {assignment: conv_assignment}).then((response) => { + standardStatusHandler(response, enqueueSnackbar); + }).catch((error) => enqueueSnackbar(error.toString(), {variant: 'error'})); + return; + } + + enqueueSnackbar('An error occurred', {variant: 'error'}); + }; + + if (assignment === null) { + return ; + } + + return ( + + + + Anubis + + + Assignment Management + + + + + + + + + + ); +} diff --git a/web/src/Pages/Admin/Assignment/Assignments.jsx b/web/src/Pages/Admin/Assignment/Assignments.jsx index 5f1209e54..b04611e78 100644 --- a/web/src/Pages/Admin/Assignment/Assignments.jsx +++ b/web/src/Pages/Admin/Assignment/Assignments.jsx @@ -2,128 +2,48 @@ import React, {useState} from 'react'; import {useSnackbar} from 'notistack'; import axios from 'axios'; -import Grid from '@material-ui/core/Grid'; +import {DataGrid} from '@material-ui/data-grid'; +import Paper from '@material-ui/core/Paper'; import makeStyles from '@material-ui/core/styles/makeStyles'; -import Button from '@material-ui/core/Button'; +import Grid from '@material-ui/core/Grid'; import Typography from '@material-ui/core/Typography'; -import CodeOutlinedIcon from '@material-ui/icons/CodeOutlined'; + +import VisibilityOffIcon from '@material-ui/icons/VisibilityOff'; +import VisibilityIcon from '@material-ui/icons/Visibility'; +import green from '@material-ui/core/colors/green'; +import grey from '@material-ui/core/colors/grey'; import standardStatusHandler from '../../../Utils/standardStatusHandler'; import ManagementIDEDialog from '../../../Components/Admin/IDE/ManagementIDEDialog'; -import AssignmentCard from '../../../Components/Admin/Assignment/AssignmentCard'; +import {Tooltip} from '@material-ui/core'; +import {Redirect} from 'react-router-dom'; const useStyles = makeStyles((theme) => ({ - root: { - minWidth: 275, - }, - bullet: { - display: 'inline-block', - margin: '0 2px', - transform: 'scale(0.8)', - }, - title: { - fontSize: 14, - }, - pos: { - marginBottom: 12, - }, - button: { - margin: theme.spacing(1), + paper: { + height: 700, + padding: theme.spacing(1), }, })); -const nonStupidDatetimeFormat = (date) => ( - `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()} ` + - `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}` -); - -const editableFields = [ - {field: 'name', label: 'Assignment Name'}, - {field: 'github_classroom_url', label: 'Github Classroom URL'}, - {field: 'theia_image', label: 'Theia Image'}, - {field: 'theia_options', label: 'Theia Options', type: 'json'}, - {field: 'pipeline_image', label: 'Pipeline Image', disabled: true}, - {field: 'unique_code', label: 'Unique Code', disabled: true}, - {field: 'hidden', label: 'Hidden', type: 'boolean'}, - {field: 'ide_enabled', label: 'Theia Enabled', type: 'boolean'}, - {field: 'autograde_enabled', label: 'Autograde Enabled', type: 'boolean'}, - {field: 'release_date', label: 'Release Date', type: 'datetime'}, - {field: 'due_date', label: 'Due Date', type: 'datetime'}, - {field: 'grace_date', label: 'Grace Date', type: 'datetime'}, -]; - - export default function Assignments() { const classes = useStyles(); const {enqueueSnackbar} = useSnackbar(); const [assignments, setAssignments] = useState([]); - const [edits, setEdits] = useState(0); - const [reset, setReset] = useState(0); - const [dialogOpen, setDialogOpen] = useState(false); + const [redirect, setRedirect] = useState(null); React.useEffect(() => { axios.get('/api/admin/assignments/list').then((response) => { const data = standardStatusHandler(response, enqueueSnackbar); if (data?.assignments) { - for (const assignment of data.assignments) { - for (const field of editableFields) { - if (field.type === 'datetime') { - assignment[field.field] = new Date(assignment[field.field].replace(/-/g, '/')); - } - } - } setAssignments(data.assignments); } }).catch((error) => enqueueSnackbar(error.toString(), {variant: 'error'})); - }, [reset]); - - const updateField = (id, field, toggle = false, datetime = false, json = false) => (e) => { - if (!e) { - return; - } - - for (const assignment of assignments) { - if (assignment.id === id) { - if (toggle) { - assignment[field] = !assignment[field]; - break; - } + }, []); - if (datetime) { - assignment[field] = e; - break; - } + if (redirect !== null) { + return ; + } - if (json) { - assignment[field] = e; - } - - assignment[field] = e.target.value.toString(); - break; - } - } - setAssignments(assignments); - setEdits((state) => ++state); - }; - - const saveAssignment = (id) => () => { - for (const assignment of assignments) { - if (assignment.id === id) { - const conv_assignment = { - ...assignment, - release_date: nonStupidDatetimeFormat(assignment.release_date), - due_date: nonStupidDatetimeFormat(assignment.due_date), - grace_date: nonStupidDatetimeFormat(assignment.grace_date), - }; - axios.post(`/api/admin/assignments/save`, {assignment: conv_assignment}).then((response) => { - standardStatusHandler(response, enqueueSnackbar); - }).catch((error) => enqueueSnackbar(error.toString(), {variant: 'error'})); - return; - } - } - - enqueueSnackbar('An error occurred', {variant: 'error'}); - }; return ( @@ -138,16 +58,24 @@ export default function Assignments() { - {assignments.map((assignment) => ( - - - - ))} + + + ( + + { + row.hidden ? + : + + } + + )}, + {field: 'release_date', headerName: 'Release Date', width: 170}, + {field: 'due_date', headerName: 'Due Date', width: 170}, + ]} rows={assignments} onRowClick={({row}) => setRedirect(`/admin/assignment/edit/${row.id}`)}/> + + ); } diff --git a/web/src/Pages/Admin/Assignment/LateExceptions.jsx b/web/src/Pages/Admin/Assignment/LateExceptions.jsx new file mode 100644 index 000000000..e90739c69 --- /dev/null +++ b/web/src/Pages/Admin/Assignment/LateExceptions.jsx @@ -0,0 +1,95 @@ +import React, {useState} from 'react'; +import {useSnackbar} from 'notistack'; +import axios from 'axios'; +import {useParams} from 'react-router-dom'; + +import {DataGrid} from '@material-ui/data-grid'; +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; +import Typography from '@material-ui/core/Typography'; +import Paper from '@material-ui/core/Paper'; +import CircularProgress from '@material-ui/core/CircularProgress'; +import Tooltip from '@material-ui/core/Tooltip'; + +import DeleteForeverIcon from '@material-ui/icons/DeleteForever'; +import IconButton from '@material-ui/core/IconButton'; + +import standardErrorHandler from '../../../Utils/standardErrorHandler'; +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import LateExceptionAddCard from '../../../Components/Admin/Assignment/LateExceptionAddCard'; + +const useStyles = makeStyles((theme) => ({ + paper: { + height: 700, + padding: theme.spacing(1), + }, +})); + +export default function LateExceptions() { + const classes = useStyles(); + const {enqueueSnackbar} = useSnackbar(); + const {assignmentId} = useParams(); + const [assignment, setAssignment] = useState(null); + const [exceptions, setExceptions] = useState([]); + const [reset, setReset] = useState(0); + + React.useEffect(() => { + axios.get(`/api/admin/late-exceptions/list/${assignmentId}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.assignment) { + setAssignment(data.assignment); + } + if (data?.late_exceptions) { + setExceptions(data.late_exceptions.map((item) => { + item.id = item.user_id; + return item; + })); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }, [reset]); + + const remove = ({assignment_id, user_id}) => () => { + axios.get(`/api/admin/late-exceptions/remove/${assignment_id}/${user_id}`).then((response) => { + standardStatusHandler(response, enqueueSnackbar); + setReset((prev) => ++prev); + }).catch(standardErrorHandler(enqueueSnackbar)); + }; + + + if (assignment === null) { + return ; + } + + return ( + + + + Anubis + + + Late Exceptions for {assignment.name} + + + + + + + + ( + + + + + + )}, + ]} rows={exceptions}/> + + + + ); +} diff --git a/web/src/Pages/Admin/Assignment/AssignmentQuestions.jsx b/web/src/Pages/Admin/Assignment/Questions.jsx similarity index 98% rename from web/src/Pages/Admin/Assignment/AssignmentQuestions.jsx rename to web/src/Pages/Admin/Assignment/Questions.jsx index fb399c159..4ddce8b07 100644 --- a/web/src/Pages/Admin/Assignment/AssignmentQuestions.jsx +++ b/web/src/Pages/Admin/Assignment/Questions.jsx @@ -44,6 +44,7 @@ export default function AssignmentQuestions() { }; const saveQuestion = (index) => () => { + console.log(questions[index]); const question = questions[index]; axios.post(`/api/admin/questions/update/${question.id}`, {question}).then((response) => { standardStatusHandler(response, enqueueSnackbar); diff --git a/web/src/Pages/Admin/Assignment/Repos.jsx b/web/src/Pages/Admin/Assignment/Repos.jsx new file mode 100644 index 000000000..b08d9bbc5 --- /dev/null +++ b/web/src/Pages/Admin/Assignment/Repos.jsx @@ -0,0 +1,86 @@ +import React, {useEffect, useState} from 'react'; +import axios from 'axios'; +import {useSnackbar} from 'notistack'; +import {useParams} from 'react-router-dom'; + +import makeStyles from '@material-ui/core/styles/makeStyles'; +import Grid from '@material-ui/core/Grid'; +import Typography from '@material-ui/core/Typography'; +import Button from '@material-ui/core/Button'; +import Dialog from '@material-ui/core/Dialog'; +import DialogActions from '@material-ui/core/DialogActions'; +import DialogContent from '@material-ui/core/DialogContent'; +import DialogContentText from '@material-ui/core/DialogContentText'; +import DialogTitle from '@material-ui/core/DialogTitle'; + +import standardErrorHandler from '../../../Utils/standardErrorHandler'; +import standardStatusHandler from '../../../Utils/standardStatusHandler'; +import AssignmentReposTable from '../../../Components/Admin/Assignment/AssignmentReposTable'; +import RepoCommandDialog from '../../../Components/Admin/Assignment/RepoCommandDialog'; +import downloadTextFile from '../../../Utils/downloadTextFile'; + +const useStyles = makeStyles((theme) => ({ + root: { + flex: 1, + }, + button: { + marginTop: theme.spacing(1), + marginBottom: theme.spacing(1), + }, +})); + + +export default function Repos() { + const classes = useStyles(); + const match = useParams(); + const {enqueueSnackbar} = useSnackbar(); + const [repos, setRepos] = useState([]); + const [assignment, setAssignment] = useState({}); + + const assignmentId = match?.assignmentId; + + useEffect(() => { + axios.get(`/api/admin/assignments/repos/${assignmentId}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.repos) { + setRepos(data?.repos); + } + if (data?.assignment) { + setAssignment(data?.assignment); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }, []); + + return ( + + + + Anubis + + + Assignment Repos + + + + + + + + + ); +} diff --git a/web/src/Pages/Admin/Assignment/AssignmentTests.jsx b/web/src/Pages/Admin/Assignment/Tests.jsx similarity index 99% rename from web/src/Pages/Admin/Assignment/AssignmentTests.jsx rename to web/src/Pages/Admin/Assignment/Tests.jsx index e9f0a9b03..be06f2a46 100644 --- a/web/src/Pages/Admin/Assignment/AssignmentTests.jsx +++ b/web/src/Pages/Admin/Assignment/Tests.jsx @@ -37,7 +37,7 @@ const useStyles = makeStyles((theme) => ({ }, })); -export default function AssignmentTests() { +export default function Tests() { const classes = useStyles(); const {enqueueSnackbar} = useSnackbar(); const match = useParams(); diff --git a/web/src/Pages/Admin/Autograde/AutogradeResults.jsx b/web/src/Pages/Admin/Autograde/Assignments.jsx similarity index 98% rename from web/src/Pages/Admin/Autograde/AutogradeResults.jsx rename to web/src/Pages/Admin/Autograde/Assignments.jsx index 4a1327863..36cd677fd 100644 --- a/web/src/Pages/Admin/Autograde/AutogradeResults.jsx +++ b/web/src/Pages/Admin/Autograde/Assignments.jsx @@ -44,7 +44,7 @@ const sortModel = [ }, ]; -export default function AutogradeResults() { +export default function Assignments() { const classes = useStyles(); const {enqueueSnackbar} = useSnackbar(); const [assignments, setAssignments] = useState([]); diff --git a/web/src/Pages/Admin/Autograde/AutogradeAssignments.jsx b/web/src/Pages/Admin/Autograde/Results.jsx similarity index 87% rename from web/src/Pages/Admin/Autograde/AutogradeAssignments.jsx rename to web/src/Pages/Admin/Autograde/Results.jsx index cc9b539e9..a2073a924 100644 --- a/web/src/Pages/Admin/Autograde/AutogradeAssignments.jsx +++ b/web/src/Pages/Admin/Autograde/Results.jsx @@ -21,6 +21,8 @@ import useQuery from '../../../hooks/useQuery'; import standardStatusHandler from '../../../Utils/standardStatusHandler'; import standardErrorHandler from '../../../Utils/standardErrorHandler'; import AutogradeVisuals from '../../../Components/Admin/Visuals/AutogradeVisuals'; +import Button from '@material-ui/core/Button'; +import RefreshIcon from '@material-ui/icons/Refresh'; const useStyles = makeStyles((theme) => ({ paper: { @@ -84,7 +86,7 @@ const useColumns = () => ([ ]); -export default function AutogradeAssignments() { +export default function Results() { const query = useQuery(); const classes = useStyles(); const {enqueueSnackbar} = useSnackbar(); @@ -167,6 +169,29 @@ export default function AutogradeAssignments() { return ; } + const clearCache = () => { + axios.get(`/api/admin/autograde/cache-reset/${assignmentId}`).then((response) => { + const data = standardStatusHandler(response, enqueueSnackbar); + if (data) { + enqueueSnackbar( + 'Cache is reset. You will likely need to reload', { + variant: 'warning', + action: ( + + ), + }); + } + }).catch(standardErrorHandler(enqueueSnackbar)); + }; + return ( @@ -179,6 +204,13 @@ export default function AutogradeAssignments() { Results for {assignment?.name} + diff --git a/web/src/Pages/Admin/Autograde/AutogradeSubmission.jsx b/web/src/Pages/Admin/Autograde/Submission.jsx similarity index 93% rename from web/src/Pages/Admin/Autograde/AutogradeSubmission.jsx rename to web/src/Pages/Admin/Autograde/Submission.jsx index 940b09931..8ca3dcc85 100644 --- a/web/src/Pages/Admin/Autograde/AutogradeSubmission.jsx +++ b/web/src/Pages/Admin/Autograde/Submission.jsx @@ -18,6 +18,7 @@ import SubmissionBuild from '../../../Components/Public/Submission/SubmissionBui import SubmissionTests from '../../../Components/Public/Submission/SubmissionTests'; import QuestionsCard from '../../../Components/Public/Questions/QuestionsCard'; import StudentGitCard from '../../../Components/Admin/Stats/Submissions/StudentGitCard'; +import StudentAssignmentHistory from '../../../Components/Admin/Visuals/StudentAssignmentHistory'; const useStyles = makeStyles((theme) => ({ @@ -26,7 +27,7 @@ const useStyles = makeStyles((theme) => ({ }, })); -export default function AutogradeSubmission() { +export default function Submission() { const classes = useStyles(); const query = useQuery(); const {enqueueSnackbar} = useSnackbar(); @@ -68,6 +69,9 @@ export default function AutogradeSubmission() { + + + diff --git a/web/src/Pages/Admin/User.jsx b/web/src/Pages/Admin/User.jsx index 32a79702c..aa7713d16 100644 --- a/web/src/Pages/Admin/User.jsx +++ b/web/src/Pages/Admin/User.jsx @@ -1,6 +1,7 @@ import React, {useState} from 'react'; import Grid from '@material-ui/core/Grid'; +import {DataGrid} from '@material-ui/data-grid'; import useQuery from '../../hooks/useQuery'; import UserCard from '../../Components/Admin/Users/UserCard'; @@ -9,33 +10,45 @@ import axios from 'axios'; import standardStatusHandler from '../../Utils/standardStatusHandler'; import {useSnackbar} from 'notistack'; import Typography from '@material-ui/core/Typography'; +import Paper from '@material-ui/core/Paper'; +import IconButton from '@material-ui/core/IconButton'; +import CheckIcon from '@material-ui/icons/Check'; +import CancelIcon from '@material-ui/icons/Cancel'; +import GitHubIcon from '@material-ui/icons/GitHub'; export default function User() { const query = useQuery(); const {enqueueSnackbar} = useSnackbar(); + const [courses, setCourses] = useState([]); + const [repos, setRepos] = useState([]); + const [theia, setTheia] = useState([]); const [user, setUser] = useState(null); - const [edits, setEdits] = useState(0); React.useEffect(() => { axios.get(`/api/admin/students/info/${query.get('userId')}`).then((response) => { - if (standardStatusHandler(response, enqueueSnackbar)) { - setUser(response?.data?.data); + const data = standardStatusHandler(response, enqueueSnackbar); + if (data?.user) { + setUser(data.user); + } + if (data?.courses) { + setCourses(data.courses); + } + if (data?.repos) { + setRepos(data.repos); + } + if (data?.theia) { + setTheia(data.theia); } }); - }, [edits]); + }, []); if (!user) { return null; } - const pageState = { - user, setUser, - edits, setEdits, - }; - return ( - + Anubis @@ -44,16 +57,88 @@ export default function User() { Student Management - - - - - - {user.courses.map((course) => ( - - - - ))} + + + + + {/* Student */} + + + Student + + + + + {/* Courses */} + + + Courses + + {courses.map((course) => ( + + ))} + + + {/* Repos */} + + + Repos + + + ( + + {params.row?.repo_url} + + ), + }, + {field: 'assignment_name', headerName: 'Assignment', width: 200}, + {field: 'course_code', headerName: 'Course Code', width: 150}, + ]} + rows={repos} + /> + + + + {/* Theia */} + + + Recent IDEs + + + ( + + {params.row.autosave ? : } + + ), + }, + { + field: 'repo_url', headerName: 'Repo', width: 100, renderCell: ({row}) => ( + + + + ), + }, + ]} + rows={theia} + /> + + + diff --git a/web/src/Pages/Admin/Users.jsx b/web/src/Pages/Admin/Users.jsx index 5fcd58064..8bfec0bfc 100644 --- a/web/src/Pages/Admin/Users.jsx +++ b/web/src/Pages/Admin/Users.jsx @@ -10,13 +10,14 @@ import Paper from '@material-ui/core/Paper'; import Grid from '@material-ui/core/Grid'; import Switch from '@material-ui/core/Switch'; import TextField from '@material-ui/core/TextField'; -import PersonIcon from '@material-ui/icons/Person'; import Fab from '@material-ui/core/Fab'; import Tooltip from '@material-ui/core/Tooltip'; -import ExitToAppIcon from '@material-ui/icons/ExitToApp'; import Autocomplete from '@material-ui/lab/Autocomplete'; import Typography from '@material-ui/core/Typography'; +import VisibilityIcon from '@material-ui/icons/Visibility'; +import ExitToAppIcon from '@material-ui/icons/ExitToApp'; + import standardStatusHandler from '../../Utils/standardStatusHandler'; import standardErrorHandler from '../../Utils/standardErrorHandler'; import AuthContext from '../../Contexts/AuthContext'; @@ -60,7 +61,7 @@ const toggleSuperuser = (id, {setStudents, setEdits}, enqueueSnackbar) => () => const useColumns = (pageState, enqueueSnackbar) => (user) => ([ { field: 'id', - headerName: 'ID', + headerName: 'View', renderCell: (params) => ( (user) => ([ component={Link} to={`/admin/user?userId=${params.row.id}`} > - + ), }, + {field: 'netid', headerName: 'netid'}, + {field: 'name', headerName: 'Name', width: 150}, + {field: 'github_username', headerName: 'Github Username', width: 200}, { field: 'log_in_as', headerName: 'Log in as', width: 130, + hide: !user.is_superuser, renderCell: (params) => ( (user) => ([ ), }, - {field: 'netid', headerName: 'netid'}, - {field: 'name', headerName: 'Name', width: 150}, - {field: 'github_username', headerName: 'Github Username', width: 200}, { field: 'is_superuser', headerName: 'Superuser', diff --git a/web/src/Pages/Public/Submissions.jsx b/web/src/Pages/Public/Submissions.jsx index b5f693596..168140f7c 100644 --- a/web/src/Pages/Public/Submissions.jsx +++ b/web/src/Pages/Public/Submissions.jsx @@ -1,103 +1,177 @@ import React, {useState} from 'react'; -import {makeStyles} from '@material-ui/core/styles'; -import Grid from '@material-ui/core/Grid'; +import {useSnackbar} from 'notistack'; +import {Redirect} from 'react-router-dom'; +import axios from 'axios'; + +import {DataGrid} from '@material-ui/data-grid'; +import makeStyles from '@material-ui/core/styles/makeStyles'; import Typography from '@material-ui/core/Typography'; -import {SubmissionsTable} from '../../Components/Public/Submissions/SubmissionsTable'; +import Paper from '@material-ui/core/Paper'; +import Grid from '@material-ui/core/Grid'; + +import CheckCircleIcon from '@material-ui/icons/CheckCircle'; +import CancelIcon from '@material-ui/icons/Cancel'; +import green from '@material-ui/core/colors/green'; +import red from '@material-ui/core/colors/red'; + import useQuery from '../../hooks/useQuery'; -import Questions from '../../Components/Public/Questions/Questions'; -import axios from 'axios'; import standardStatusHandler from '../../Utils/standardStatusHandler'; -import {useSnackbar} from 'notistack'; -import AuthContext from '../../Contexts/AuthContext'; +import standardErrorHandler from '../../Utils/standardErrorHandler'; +import Questions from '../../Components/Public/Questions/Questions'; -const useStyles = makeStyles({ - root: { - flexGrow: 1, - }, - table: { - minWidth: 500, +const useStyles = makeStyles((theme) => ({ + paper: { + height: 700, + padding: theme.spacing(1), }, - headerText: { - fontWeight: 600, - }, - commitHashContainer: { - width: 200, - overflow: 'hidden', - }, -}); - -function translateSubmission({assignment_name, assignment_due, commit, processed, state, created, tests}) { - const testsPassed = tests.filter((test) => test.result.passed).length; - const totalTests = tests.length; +})); +function translateSubmission({id, assignment_name, assignment_due, commit, processed, state, created, tests}) { return { - assignmentName: assignment_name, assignmentDue: new Date(assignment_due), state: state, - commitHash: commit, processed: processed, timeSubmitted: created.split(' ')[0], - dateSubmitted: created.split(' ')[1], timeStamp: new Date(created), testsPassed, totalTests, + assignment_name, assignment_due, commit, created, tests, + id, assignmentDue: new Date(assignment_due), state: state, + processed: processed, timeStamp: new Date(created), }; } +const useColumns = () => ([ + {field: 'id', hide: true}, + {field: 'assignment_name', headerName: 'Assignment Name', width: 200}, + { + field: 'commit', headerName: 'Commit Hash', width: 150, renderCell: ({row}) => ( + row?.commit && row.commit.substring(0, 10) + ), + }, + { + field: 'tests_passed', headerName: 'Tests Passed', width: 150, renderCell: ({row}) => ( + row?.tests && `${row.tests.filter((test) => test.result.passed).length}/${row.tests.length}` + ), + }, + { + field: 'processed', headerName: 'Processed', width: 150, renderCell: ({row}) => ( + row.processed && (row.processed ? + : + ) + ), + }, + { + field: 'on_time', headerName: 'On Time', width: 150, renderCell: ({row}) => ( + row?.timeStamp && (row.timeStamp <= row.assignmentDue ? + : + ) + ), + }, + { + field: 'timeStamp', headerName: 'Timestamp', width: 250, renderCell: ({row}) => ( + row.created + ), + }, +]); + + export default function Submissions() { const classes = useStyles(); const query = useQuery(); + const columns = useColumns(); const {enqueueSnackbar} = useSnackbar(); - const [submissions, setSubmissions] = useState([]); + const [rows, setRows] = useState([]); + const [user, setUser] = useState(null); + const [loading, setLoading] = useState(true); + const [rowCount, setRowCount] = useState(0); + const [page, setPage] = useState(0); + const [pageSize, setPageSize] = useState(10); + const [redirect, setRedirect] = useState(null); - const assignment_id = query.get('assignmentId'); + const assignmentId = query.get('assignmentId'); + const courseId = query.get('courseId'); + const userId = query.get('userId'); React.useEffect(() => { - axios.get( - `/api/public/submissions/`, - { - params: { - assignmentId: query.get('assignmentId'), - courseId: query.get('courseId'), - userId: query.get('userId'), - }, + setLoading(true); + axios.get(`/api/public/submissions/`, { + params: { + assignmentId, courseId, userId, + limit: pageSize, + offset: page * pageSize, }, - ).then((response) => { + }).then((response) => { const data = standardStatusHandler(response, enqueueSnackbar); - setSubmissions(data.submissions - .map(translateSubmission) - .sort((a, b) => (a.timeStamp > b.timeStamp ? -1 : 1))); - }).catch((error) => enqueueSnackbar(error.toString(), {variant: 'error'})); - }, []); + if (data?.total) { + setRowCount(data?.total); + } + if (data?.submissions) { + let prev; + if (rows.length === 0) { + prev = new Array(data?.total); + for (let i = 0; i < prev.length; ++i) { + prev[i] = {id: i}; + } + } else { + prev = rows; + } - return ( -
- - - - Anubis - - - {(user) => ( - - {user?.name}'s Submissions - - )} - - + const translation = data.submissions.map(translateSubmission); + for (let i = page * pageSize; i < (page * pageSize) + translation.length; ++i) { + prev[i] = translation[i - (page * pageSize)]; + } + + setRows([...prev]); + } + if (data?.user) { + setUser(data.user); + } + setLoading(false); + }).catch(standardErrorHandler(enqueueSnackbar)); + }, [pageSize, page]); - + if (redirect !== null) { + return ; + } - - - {/* Questions */} - {!!assignment_id ? ( - - - - ) : null} + return ( + + + + Anubis + + + {user && `${user?.name}'s Submissions`} + + - {/* Table */} - - + + + + + {/* Questions */} + {!!assignmentId ? ( + + + ) : null} + + {/* Table */} + + + setPage(page)} + onPageSizeChange={({pageSize}) => setPageSize(pageSize)} + onRowClick={({row}) => setRedirect(`/submission?commit=${row.commit}`)} + columns={columns} + rows={rows} + rowCount={rowCount} + loading={loading} + /> + + -
+ ); } diff --git a/web/src/Pages/Public/Visuals.jsx b/web/src/Pages/Public/Visuals.jsx index e97197070..3551d42ae 100644 --- a/web/src/Pages/Public/Visuals.jsx +++ b/web/src/Pages/Public/Visuals.jsx @@ -56,6 +56,21 @@ export default function Visuals() { + + + } + title={'Anubis Usage Over Time'} + titleTypographyProps={{variant: 'h6'}} + subheader={'re-generated every 5 minutes'} + /> + + +
@@ -77,21 +92,6 @@ export default function Visuals() {
- - - } - title={'Anubis Usage Over Time'} - titleTypographyProps={{variant: 'h6'}} - subheader={'re-generated every 5 minutes'} - /> - - - ); } diff --git a/web/src/Utils/datetime.js b/web/src/Utils/datetime.js new file mode 100644 index 000000000..a69c35037 --- /dev/null +++ b/web/src/Utils/datetime.js @@ -0,0 +1,8 @@ +export const nonStupidDatetimeFormat = (date) => { + let seconds = date.getSeconds().toString(); + if (seconds.length === 1) { + seconds = '0' + seconds; + } + return `${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()} ` + + `${date.getHours()}:${date.getMinutes()}:${seconds}`; +}; diff --git a/web/src/navconfig.jsx b/web/src/navconfig.jsx index 6d7f8a2ad..7e2cf87a3 100644 --- a/web/src/navconfig.jsx +++ b/web/src/navconfig.jsx @@ -26,16 +26,19 @@ import Visuals from './Pages/Public/Visuals'; import AdminUsers from './Pages/Admin/Users'; import AdminUser from './Pages/Admin/User'; -import AdminAssignments from './Pages/Admin/Assignment/Assignments'; -import AdminStats from './Pages/Admin/Autograde/AutogradeResults'; -import AdminAssignmentStats from './Pages/Admin/Autograde/AutogradeAssignments'; -import AdminSubmissionStats from './Pages/Admin/Autograde/AutogradeSubmission'; import AdminCourse from './Pages/Admin/Course'; import AdminTheia from './Pages/Admin/Theia'; import AdminStatic from './Pages/Admin/Static'; import AdminConfig from './Pages/Admin/Config'; -import AdminAssignmentQuestions from './Pages/Admin/Assignment/AssignmentQuestions'; -import AdminAssignmentTests from './Pages/Admin/Assignment/AssignmentTests'; +import AdminAutogradeAssignments from './Pages/Admin/Autograde/Assignments'; +import AdminAutogradeSubmission from './Pages/Admin/Autograde/Submission'; +import AdminAutogradeResults from './Pages/Admin/Autograde/Results'; +import AdminAssignmentLateExceptions from './Pages/Admin/Assignment/LateExceptions'; +import AdminAssignmentAssignments from './Pages/Admin/Assignment/Assignments'; +import AdminAssignmentAssignment from './Pages/Admin/Assignment/Assignment'; +import AdminAssignmentQuestions from './Pages/Admin/Assignment/Questions'; +import AdminAssignmentTests from './Pages/Admin/Assignment/Tests'; +import AdminAssignmentRepos from './Pages/Admin/Assignment/Repos'; export const footer_nav = [ { @@ -118,13 +121,13 @@ export const admin_nav = [ id: 'Assignments', icon: , path: '/admin/assignments', - Page: AdminAssignments, + Page: AdminAssignmentAssignments, }, { id: 'Autograde Results', icon: , path: '/admin/autograde', - Page: AdminStats, + Page: AdminAutogradeAssignments, }, { id: 'Anubis Cloud IDE', @@ -160,12 +163,12 @@ export const not_shown_nav = [ { id: 'AdminAssignmentStats', path: '/admin/autograde/assignment', - Page: AdminAssignmentStats, + Page: AdminAutogradeResults, }, { id: 'AdminSubmissionStats', path: '/admin/autograde/submission', - Page: AdminSubmissionStats, + Page: AdminAutogradeSubmission, }, { id: '', @@ -177,6 +180,21 @@ export const not_shown_nav = [ path: '/admin/assignment/tests/:assignmentId', Page: AdminAssignmentTests, }, + { + id: '', + path: '/admin/assignment/repos/:assignmentId', + Page: AdminAssignmentRepos, + }, + { + id: '', + path: '/admin/assignment/edit/:assignmentId', + Page: AdminAssignmentAssignment, + }, + { + id: '', + path: '/admin/assignment/late-exceptions/:assignmentId', + Page: AdminAssignmentLateExceptions, + }, ]; export const drawerWidth = 240;