From fff075d220bffa87f8a9ad5ad0f87de6dcc5b376 Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Thu, 28 Nov 2024 11:09:33 +0700 Subject: [PATCH 01/79] feat: add dynamic form generation --- SORT/settings.py | 1 - home/urls.py | 1 - home/views.py | 4 +- invites/__init__.py | 0 invites/admin.py | 3 - invites/apps.py | 6 - invites/forms.py | 9 - invites/migrations/0001_initial.py | 39 --- .../migrations/0002_alter_invitation_token.py | 19 -- invites/migrations/__init__.py | 0 invites/models.py | 22 -- invites/tests.py | 3 - invites/urls.py | 7 - invites/views.py | 49 ---- requirements.txt | 1 + survey/admin.py | 7 +- survey/forms.py | 100 ++++--- survey/migrations/0001_initial.py | 81 ++---- ...2_answer_submitted_at_alter_answer_user.py | 27 -- .../0003_alter_question_question_type.py | 26 -- .../0004_alter_question_questionnaire.py | 22 -- .../migrations/0005_question_answer_type.py | 26 -- .../0006_remove_question_answer_type.py | 16 - survey/migrations/0007_comment.py | 45 --- ...move_comment_user_answer_token_and_more.py | 34 --- ..._alter_answer_token_alter_comment_token.py | 22 -- survey/misc.py | 73 +++++ survey/mixins.py | 2 +- survey/models.py | 61 ++-- .../invitations/complete_invitation.html | 0 .../invitations/send_invitation.html | 0 survey/templates/survey/survey.html | 17 ++ .../survey/survey_link_invalid_view.html | 8 + survey/templates/survey/survey_response.html | 13 + survey/urls.py | 11 +- survey/views.py | 274 ++++++++++++------ 36 files changed, 417 insertions(+), 612 deletions(-) delete mode 100644 invites/__init__.py delete mode 100644 invites/admin.py delete mode 100644 invites/apps.py delete mode 100644 invites/forms.py delete mode 100644 invites/migrations/0001_initial.py delete mode 100644 invites/migrations/0002_alter_invitation_token.py delete mode 100644 invites/migrations/__init__.py delete mode 100644 invites/models.py delete mode 100644 invites/tests.py delete mode 100644 invites/urls.py delete mode 100644 invites/views.py delete mode 100644 survey/migrations/0002_answer_submitted_at_alter_answer_user.py delete mode 100644 survey/migrations/0003_alter_question_question_type.py delete mode 100644 survey/migrations/0004_alter_question_questionnaire.py delete mode 100644 survey/migrations/0005_question_answer_type.py delete mode 100644 survey/migrations/0006_remove_question_answer_type.py delete mode 100644 survey/migrations/0007_comment.py delete mode 100644 survey/migrations/0008_remove_answer_user_remove_comment_user_answer_token_and_more.py delete mode 100644 survey/migrations/0009_alter_answer_token_alter_comment_token.py create mode 100644 survey/misc.py rename {invites => survey}/templates/invitations/complete_invitation.html (100%) rename {invites => survey}/templates/invitations/send_invitation.html (100%) create mode 100644 survey/templates/survey/survey.html create mode 100644 survey/templates/survey/survey_link_invalid_view.html create mode 100644 survey/templates/survey/survey_response.html diff --git a/SORT/settings.py b/SORT/settings.py index fd7cf4c..3904b92 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -48,7 +48,6 @@ "home", "survey", - "invites", ] MIDDLEWARE = [ diff --git a/home/urls.py b/home/urls.py index 1d1ad0c..c6c6076 100644 --- a/home/urls.py +++ b/home/urls.py @@ -9,7 +9,6 @@ path('logout/', views.LogoutInterfaceView.as_view(), name='logout'), path('signup/', views.SignupView.as_view(), name='signup'), path('', include('survey.urls'), name='survey'), - path('invite/', include('invites.urls'), name='invites'), path('profile/', views.ProfileView.as_view(), name='profile'), path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), path('password_reset/done/', views.CustomPasswordResetDoneView.as_view(), name='password_reset_done'), diff --git a/home/views.py b/home/views.py index cb58a55..555a2d4 100644 --- a/home/views.py +++ b/home/views.py @@ -3,7 +3,7 @@ from django.contrib.auth.views import LoginView, LogoutView from django.views.generic.edit import CreateView, UpdateView from django.shortcuts import redirect -from survey.models import Questionnaire +from survey.models import Survey from django.shortcuts import render from django.views import View from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm @@ -44,7 +44,7 @@ class HomeView(LoginRequiredMixin, View): template_name = 'home/welcome.html' login_url = 'login' def get(self, request): - consent_questionnaire = Questionnaire.objects.get( + consent_questionnaire = Survey.objects.get( title="Consent") return render(request, 'home/welcome.html', {'questionnaire': consent_questionnaire}) diff --git a/invites/__init__.py b/invites/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/invites/admin.py b/invites/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/invites/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/invites/apps.py b/invites/apps.py deleted file mode 100644 index c7429fb..0000000 --- a/invites/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class InvitationsConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "invites" diff --git a/invites/forms.py b/invites/forms.py deleted file mode 100644 index 058f076..0000000 --- a/invites/forms.py +++ /dev/null @@ -1,9 +0,0 @@ -from django import forms -from django.core.validators import EmailValidator - - -class InvitationForm(forms.Form): - email = forms.EmailField(label='Participant Email', - max_length=100, - required=True, - validators=[EmailValidator()]) \ No newline at end of file diff --git a/invites/migrations/0001_initial.py b/invites/migrations/0001_initial.py deleted file mode 100644 index 48b030c..0000000 --- a/invites/migrations/0001_initial.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-22 14:33 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("survey", "0007_comment"), - ] - - operations = [ - migrations.CreateModel( - name="Invitation", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("token", models.CharField(blank=True, max_length=64, unique=True)), - ("created_at", models.DateTimeField(auto_now_add=True)), - ("used", models.BooleanField(default=False)), - ( - "questionnaire", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="survey.questionnaire", - ), - ), - ], - ), - ] diff --git a/invites/migrations/0002_alter_invitation_token.py b/invites/migrations/0002_alter_invitation_token.py deleted file mode 100644 index 874b843..0000000 --- a/invites/migrations/0002_alter_invitation_token.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-23 09:03 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("invites", "0001_initial"), - ] - - operations = [ - migrations.AlterField( - model_name="invitation", - name="token", - field=models.CharField( - blank=True, editable=False, max_length=64, unique=True - ), - ), - ] diff --git a/invites/migrations/__init__.py b/invites/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/invites/models.py b/invites/models.py deleted file mode 100644 index 2a09e8d..0000000 --- a/invites/models.py +++ /dev/null @@ -1,22 +0,0 @@ -import secrets -from django.db import models -from django.utils import timezone -from survey.models import Questionnaire - -class Invitation(models.Model): - - questionnaire = models.ForeignKey(Questionnaire, on_delete=models.CASCADE) - token = models.CharField(max_length=64, unique=True, blank=True, editable=False) - created_at = models.DateTimeField(auto_now_add=True) - used = models.BooleanField(default=False) - - def __str__(self): - return f"Invitation for {self.questionnaire.title}" - - def save(self, *args, **kwargs): - if not self.token: - self.token = secrets.token_urlsafe(32) - super().save(*args, **kwargs) - - def is_expired(self): - return timezone.now() > self.created_at + timezone.timedelta(days=7) diff --git a/invites/tests.py b/invites/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/invites/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/invites/urls.py b/invites/urls.py deleted file mode 100644 index fc16e83..0000000 --- a/invites/urls.py +++ /dev/null @@ -1,7 +0,0 @@ -from django.urls import path -from .views import InvitationView, SuccessInvitationView - -urlpatterns = [ - path('', InvitationView.as_view(), name='invite'), - path('success/', SuccessInvitationView.as_view(), name='success_invitation') -] \ No newline at end of file diff --git a/invites/views.py b/invites/views.py deleted file mode 100644 index 7beb395..0000000 --- a/invites/views.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.views.generic.edit import FormView -from django.core.mail import send_mail -from django.contrib import messages -from .forms import InvitationForm -from django.urls import reverse_lazy -from django.views.generic import TemplateView -from survey.models import Questionnaire -from invites.models import Invitation -from django.contrib.auth.mixins import LoginRequiredMixin -from django.shortcuts import get_object_or_404, redirect -from django.http import HttpResponseForbidden - - -class InvitationView(FormView): - - template_name = 'invitations/send_invitation.html' - form_class = InvitationForm - success_url = reverse_lazy('success_invitation') - - def form_valid(self, form): - email = form.cleaned_data['email'] - - questionnaire = Questionnaire.objects.first() - - invitation = Invitation.objects.create(questionnaire=questionnaire) - - token = invitation.token - - # Generate the survey link with the token - survey_link = f"http://localhost:8000/survey/{questionnaire.pk}/{token}/" - - # Send the email - send_mail( - 'Your Survey Invitation', - f'Click here to start the survey: {survey_link}', - 'from@example.com', - [email], - fail_silently=False, - ) - - # Show success message - messages.success(self.request, f'Invitation sent to {email}.') - return super().form_valid(form) - - - -class SuccessInvitationView(LoginRequiredMixin, TemplateView): - - template_name = 'invitations/complete_invitation.html' diff --git a/requirements.txt b/requirements.txt index 053c4ad..da5b1df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ traitlets==5.14.3 typing_extensions==4.12.2 tzdata==2024.1 wcwidth==0.2.13 +strenum==0.4.15 diff --git a/survey/admin.py b/survey/admin.py index 402c11c..d85e9b7 100644 --- a/survey/admin.py +++ b/survey/admin.py @@ -1,6 +1,5 @@ from django.contrib import admin -from .models import Questionnaire, Question, Answer +from .models import Survey, SurveyResponse -admin.site.register(Questionnaire) -admin.site.register(Question) -admin.site.register(Answer) +admin.site.register(Survey) +admin.site.register(SurveyResponse) diff --git a/survey/forms.py b/survey/forms.py index ef9f0c8..e345f8a 100644 --- a/survey/forms.py +++ b/survey/forms.py @@ -1,40 +1,62 @@ from django import forms -from .models import Answer, Question -class AnswerForm(forms.ModelForm): - class Meta: - model = Answer - fields = [] # no defaults needed - - def __init__(self, *args, **kwargs): - questionnaire = kwargs.pop('questionnaire', None) # get the questionnaire passed in - super().__init__(*args, **kwargs) - - if questionnaire: - - for index, question in enumerate(questionnaire.questions.all(), start=1): - if question.question_type == 'boolean': - self.fields[f'question_{question.id}'] = forms.ChoiceField( - label=f"{index}. {question.question_text}", - choices=[('agree', 'I agree'), ('disagree', 'I disagree')], - widget=forms.RadioSelect, - required=True - ) - - elif question.question_type == 'rating': - - self.fields[f'question_{question.id}'] = forms.ChoiceField( - label=question.question_text, - choices=[(i, str(i)) for i in range(1, 6)], # assuming a 1-5 rating - required=True - ) - - - else: - - pass - - # self.fields[f'question_{question.id}'] = forms.CharField( - # label=question.question_text, - # required=True, - # widget=forms.Textarea(attrs={'rows': 4, 'style': 'width: 80%;'}), - # ) +from django.forms import BaseFormSet, formset_factory +from strenum import StrEnum +from django.core.validators import EmailValidator + +class InvitationForm(forms.Form): + email = forms.EmailField(label='Participant Email', + max_length=100, + required=True, + validators=[EmailValidator()]) + +class FormFieldType(StrEnum): + CHAR = "char" + TEXT = "text" + RADIO = "radio" + CHECKBOX = "checkbox" + LIKERT = "likert" + + + +def create_field_from_config(field_config: dict): + """ + Convert a field configuration into the correct django field + """ + if field_config['type'] == FormFieldType.CHAR: + field = forms.CharField(label=field_config["label"]) + elif field_config['type'] == FormFieldType.TEXT: + field = forms.CharField(label=field_config["label"], + widget=forms.Textarea) + elif field_config['type'] == FormFieldType.RADIO: + field = forms.ChoiceField(label=field_config["label"], + choices=field_config["options"], + widget=forms.RadioSelect) + elif field_config['type'] == FormFieldType.CHECKBOX: + field = forms.MultipleChoiceField(label=field_config["label"], + widget=forms.CheckboxSelectMultiple, + choices=field_config["options"]) + else: + field = forms.CharField(label=field_config["label"], + widget=forms.Textarea) + + if "required" in field_config: + field.required = field_config["required"] + + return field + + + +def create_dynamic_formset(field_configs: list): + """ + Create a dynamic form set from a list of field configurations. + """ + class BlankDynamicForm(forms.Form): + pass + + class BaseTestFormSet(BaseFormSet): + def add_fields(self, form, index): + super().add_fields(form, index) + for field_config in field_configs: + form.fields[field_config["name"]] = create_field_from_config(field_config) + + return formset_factory(BlankDynamicForm, BaseTestFormSet, min_num=1, max_num=1) diff --git a/survey/migrations/0001_initial.py b/survey/migrations/0001_initial.py index 9849a7b..796716e 100644 --- a/survey/migrations/0001_initial.py +++ b/survey/migrations/0001_initial.py @@ -1,85 +1,42 @@ -# Generated by Django 5.1.1 on 2024-09-12 18:18 +# Generated by Django 5.1.2 on 2024-12-03 06:54 import django.db.models.deletion from django.db import migrations, models class Migration(migrations.Migration): + initial = True - dependencies = [] + dependencies = [ + ] operations = [ migrations.CreateModel( - name="Question", + name='Survey', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("question_text", models.CharField(max_length=500)), - ( - "question_type", - models.CharField( - choices=[ - ("text", "Text"), - ("multiple_choice", "Multiple Choice"), - ("rating", "Rating"), - ], - max_length=50, - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=200)), + ('description', models.TextField()), + ('survey_config', models.JSONField()), ], ), migrations.CreateModel( - name="Questionnaire", + name='Invitation', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("title", models.CharField(max_length=200)), - ("description", models.TextField()), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('token', models.CharField(blank=True, editable=False, max_length=64, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('used', models.BooleanField(default=False)), + ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='survey.survey')), ], ), migrations.CreateModel( - name="Answer", + name='SurveyResponse', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("user", models.CharField(max_length=100)), - ("answer_text", models.TextField(blank=True, null=True)), - ( - "question", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="survey.question", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('answers', models.JSONField()), + ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='survey', to='survey.survey')), ], ), - migrations.AddField( - model_name="question", - name="questionnaire", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="survey.questionnaire" - ), - ), ] diff --git a/survey/migrations/0002_answer_submitted_at_alter_answer_user.py b/survey/migrations/0002_answer_submitted_at_alter_answer_user.py deleted file mode 100644 index b61a588..0000000 --- a/survey/migrations/0002_answer_submitted_at_alter_answer_user.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-01 13:59 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.AddField( - model_name="answer", - name="submitted_at", - field=models.DateTimeField(null=True), - ), - migrations.AlterField( - model_name="answer", - name="user", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL - ), - ), - ] diff --git a/survey/migrations/0003_alter_question_question_type.py b/survey/migrations/0003_alter_question_question_type.py deleted file mode 100644 index 1bd04d0..0000000 --- a/survey/migrations/0003_alter_question_question_type.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-01 15:43 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0002_answer_submitted_at_alter_answer_user"), - ] - - operations = [ - migrations.AlterField( - model_name="question", - name="question_type", - field=models.CharField( - choices=[ - ("text", "Text"), - ("multiple_choice", "Multiple Choice"), - ("rating", "Rating"), - ("boolean", "Agree/Disagree"), - ], - default="multiple_choice", - max_length=50, - ), - ), - ] diff --git a/survey/migrations/0004_alter_question_questionnaire.py b/survey/migrations/0004_alter_question_questionnaire.py deleted file mode 100644 index 1c35847..0000000 --- a/survey/migrations/0004_alter_question_questionnaire.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-01 16:17 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0003_alter_question_question_type"), - ] - - operations = [ - migrations.AlterField( - model_name="question", - name="questionnaire", - field=models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="questions", - to="survey.questionnaire", - ), - ), - ] diff --git a/survey/migrations/0005_question_answer_type.py b/survey/migrations/0005_question_answer_type.py deleted file mode 100644 index 8ddf9eb..0000000 --- a/survey/migrations/0005_question_answer_type.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-01 16:24 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0004_alter_question_questionnaire"), - ] - - operations = [ - migrations.AddField( - model_name="question", - name="answer_type", - field=models.CharField( - blank=True, - choices=[ - ("text", "Text"), - ("multiple_choice", "Multiple Choice"), - ("rating", "Rating"), - ("boolean", "Agree/Disagree"), - ], - max_length=20, - ), - ), - ] diff --git a/survey/migrations/0006_remove_question_answer_type.py b/survey/migrations/0006_remove_question_answer_type.py deleted file mode 100644 index d714e1a..0000000 --- a/survey/migrations/0006_remove_question_answer_type.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-01 16:40 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0005_question_answer_type"), - ] - - operations = [ - migrations.RemoveField( - model_name="question", - name="answer_type", - ), - ] diff --git a/survey/migrations/0007_comment.py b/survey/migrations/0007_comment.py deleted file mode 100644 index 08f5729..0000000 --- a/survey/migrations/0007_comment.py +++ /dev/null @@ -1,45 +0,0 @@ -# Generated by Django 5.1.1 on 2024-10-03 12:52 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0006_remove_question_answer_type"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="Comment", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("text", models.TextField()), - ("submitted_at", models.DateTimeField(null=True)), - ( - "questionnaire", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="survey.questionnaire", - ), - ), - ( - "user", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL, - ), - ), - ], - ), - ] diff --git a/survey/migrations/0008_remove_answer_user_remove_comment_user_answer_token_and_more.py b/survey/migrations/0008_remove_answer_user_remove_comment_user_answer_token_and_more.py deleted file mode 100644 index bcbab21..0000000 --- a/survey/migrations/0008_remove_answer_user_remove_comment_user_answer_token_and_more.py +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-23 09:13 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0007_comment"), - ] - - operations = [ - migrations.RemoveField( - model_name="answer", - name="user", - ), - migrations.RemoveField( - model_name="comment", - name="user", - ), - migrations.AddField( - model_name="answer", - name="token", - field=models.CharField( - blank=True, editable=False, max_length=64, unique=True - ), - ), - migrations.AddField( - model_name="comment", - name="token", - field=models.CharField( - blank=True, editable=False, max_length=64, unique=True - ), - ), - ] diff --git a/survey/migrations/0009_alter_answer_token_alter_comment_token.py b/survey/migrations/0009_alter_answer_token_alter_comment_token.py deleted file mode 100644 index 30857a2..0000000 --- a/survey/migrations/0009_alter_answer_token_alter_comment_token.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-23 09:15 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("survey", "0008_remove_answer_user_remove_comment_user_answer_token_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="answer", - name="token", - field=models.CharField(blank=True, editable=False, max_length=64), - ), - migrations.AlterField( - model_name="comment", - name="token", - field=models.CharField(blank=True, editable=False, max_length=64), - ), - ] diff --git a/survey/misc.py b/survey/misc.py new file mode 100644 index 0000000..9639b20 --- /dev/null +++ b/survey/misc.py @@ -0,0 +1,73 @@ +test_survey_config = { + "sections": [ + { + "title": "Welcome", + "type": "consent", + "fields": [ + { + "type": "radio", + "name": "consent", + "label": "Your agreement to complete the survey", + "required": True, + "options": [["Yes", "I agree to complete the survey"], ["No", "I do not agree"]] + } + ] + }, + { + "title": "Section 1", + "description": "", + "fields": [ + { + "type": "char", + "name": "char_field", + "label": "Char field", + }, + { + "type": "text", + "name": "text_field", + "label": "Text field", + }, + { + "type": "checkbox", + "name": "checkbox_field", + "label": "Checkbox field", + "options": [["Value 1", "Label 1"], ["Value 2", "Label 2"]] + }, + { + "type": "radio", + "name": "radio_field", + "label": "Radio field", + "options": [["Value 1", "Label 1"], ["Value 2", "Label 2"]] + } + ], + }, + { + "title": "Section 2", + "description": "", + "fields": [ + { + "type": "char", + "name": "char_field", + "label": "Char field", + }, + { + "type": "text", + "name": "text_field", + "label": "Text field", + }, + { + "type": "checkbox", + "name": "checkbox_field", + "label": "Checkbox field", + "options": [["Value 1", "Label 1"], ["Value 2", "Label 2"]] + }, + { + "type": "radio", + "name": "radio_field", + "label": "Radio field", + "options": [["Value 1", "Label 1"], ["Value 2", "Label 2"]] + } + ], + } + ] + } \ No newline at end of file diff --git a/survey/mixins.py b/survey/mixins.py index ab2320e..cfc6e51 100644 --- a/survey/mixins.py +++ b/survey/mixins.py @@ -1,7 +1,7 @@ __author__ = "Farhad Allian" from django.shortcuts import redirect -from invites.models import Invitation +from survey.models import Invitation class TokenAuthenticationMixin: diff --git a/survey/models.py b/survey/models.py index d6656b4..74a76f5 100644 --- a/survey/models.py +++ b/survey/models.py @@ -1,11 +1,17 @@ +import secrets from django.db import models from django.contrib.auth.models import User from django.urls import reverse +from django.utils import timezone -class Questionnaire(models.Model): - # Questionnaire data model +class Survey(models.Model): + """ + Represents a survey that will be sent out to a participant + """ title = models.CharField(max_length=200) description = models.TextField() + survey_config = models.JSONField() + # TODO: Add the project it belongs to as foreign key def __str__(self): return self.title @@ -14,40 +20,33 @@ def get_absolute_url(self, token): return reverse('survey', kwargs={'pk': self.pk, 'token': token}) -class Question(models.Model): - # Question data model - QUESTION_TYPE_CHOICES = [ - ('text', 'Text'), - ('multiple_choice', 'Multiple Choice'), - ('rating', 'Rating'), - ('boolean', 'Agree/Disagree') - ] +class SurveyResponse(models.Model): + """ + Represents a single response to the survey from a participant + """ - questionnaire = models.ForeignKey(Questionnaire, related_name='questions', on_delete=models.CASCADE) # Many questions belong to one questionnaire - question_text = models.CharField(max_length=500) - question_type = models.CharField(max_length=50, choices=QUESTION_TYPE_CHOICES, default='multiple_choice') + survey = models.ForeignKey(Survey, related_name='survey', on_delete=models.CASCADE) # Many questions belong to one survey + answers = models.JSONField() - def __str__(self): - return self.question_text + def get_absolute_url(self, token): + return reverse('survey', kwargs={'pk': self.pk, 'token': token}) -class Answer(models.Model): - # User answer data model - question = models.ForeignKey(Question, on_delete=models.CASCADE) - answer_text = models.TextField(blank=True, null=True) - token = models.CharField(max_length=64, blank=True, editable=False) - submitted_at = models.DateTimeField(null=True) # - def __str__(self): - return f"Answer for {self.question.question_text}" +class Invitation(models.Model): - -class Comment(models.Model): - # Comments data model - questionnaire = models.ForeignKey(Questionnaire, on_delete=models.CASCADE) - token = models.CharField(max_length=64, blank=True, editable=False) - text = models.TextField() - submitted_at = models.DateTimeField(null=True) + survey = models.ForeignKey(Survey, on_delete=models.CASCADE) + token = models.CharField(max_length=64, unique=True, blank=True, editable=False) + created_at = models.DateTimeField(auto_now_add=True) + used = models.BooleanField(default=False) def __str__(self): - return f"Comment on {self.questionnaire.title}" \ No newline at end of file + return f"Invitation for {self.survey.title}" + + def save(self, *args, **kwargs): + if not self.token: + self.token = secrets.token_urlsafe(32) + super().save(*args, **kwargs) + + def is_expired(self): + return timezone.now() > self.created_at + timezone.timedelta(days=7) \ No newline at end of file diff --git a/invites/templates/invitations/complete_invitation.html b/survey/templates/invitations/complete_invitation.html similarity index 100% rename from invites/templates/invitations/complete_invitation.html rename to survey/templates/invitations/complete_invitation.html diff --git a/invites/templates/invitations/send_invitation.html b/survey/templates/invitations/send_invitation.html similarity index 100% rename from invites/templates/invitations/send_invitation.html rename to survey/templates/invitations/send_invitation.html diff --git a/survey/templates/survey/survey.html b/survey/templates/survey/survey.html new file mode 100644 index 0000000..0c9ee4f --- /dev/null +++ b/survey/templates/survey/survey.html @@ -0,0 +1,17 @@ +{% extends 'base.html' %} + +{% block content %} +

{{ survey.title }}

+

{{ survey.description }}

+ + + + + + + +{% endblock %} \ No newline at end of file diff --git a/survey/templates/survey/survey_link_invalid_view.html b/survey/templates/survey/survey_link_invalid_view.html new file mode 100644 index 0000000..87d4fa2 --- /dev/null +++ b/survey/templates/survey/survey_link_invalid_view.html @@ -0,0 +1,8 @@ +{% extends 'base.html' %} + +{% block content %} + +

Your survey link is invalid

+

Reason ... e.g. token expired, no longer accepting responses,etc.

+ +{% endblock %} \ No newline at end of file diff --git a/survey/templates/survey/survey_response.html b/survey/templates/survey/survey_response.html new file mode 100644 index 0000000..7395716 --- /dev/null +++ b/survey/templates/survey/survey_response.html @@ -0,0 +1,13 @@ +{% extends 'base.html' %} + +{% block content %} + +

{{ title }}

+
+ {% csrf_token %} + {{ form }} + + +
+ +{% endblock %} \ No newline at end of file diff --git a/survey/urls.py b/survey/urls.py index a25b1d4..d570a64 100644 --- a/survey/urls.py +++ b/survey/urls.py @@ -2,6 +2,11 @@ from . import views urlpatterns = [ - path('survey///', views.QuestionnaireView.as_view(), name='questionnaire'), - path('completion/', views.CompletionView.as_view(), name='completion_page') -] \ No newline at end of file + path('survey/', views.SurveyView.as_view(), name='survey'), + path('completion/', views.CompletionView.as_view(), name='completion_page'), + path('survey_response//', views.SurveyResponseView.as_view(), name='survey_response'), + path('survey_link_invalid/', views.SurveyLinkInvalidView.as_view(), name='survey_link_invalid'), + path('invite/', views.InvitationView.as_view(), name='invite'), + path('invite/success/', views.SuccessInvitationView.as_view(), name='success_invitation'), + +] diff --git a/survey/views.py b/survey/views.py index 380fee1..0cdf23e 100644 --- a/survey/views.py +++ b/survey/views.py @@ -1,109 +1,148 @@ +from django.core.mail import send_mail +from django.http import HttpRequest from django.shortcuts import render, get_object_or_404, redirect +from django.urls import reverse_lazy from django.views import View -from django.contrib.auth.mixins import LoginRequiredMixin -from .models import Questionnaire, Answer, Comment -from .forms import AnswerForm +from django.views.generic import FormView, TemplateView +from django.views.generic.edit import CreateView from django.contrib import messages -from invites.models import Invitation +from django.contrib.auth.mixins import LoginRequiredMixin from .mixins import TokenAuthenticationMixin -import logging +from .forms import create_dynamic_formset, InvitationForm +from .models import Survey, SurveyResponse +from .models import Invitation +from .misc import test_survey_config + +import logging logger = logging.getLogger(__name__) -class QuestionnaireView(TokenAuthenticationMixin, View): +class SurveyView(LoginRequiredMixin, View): + """ + Manager's view of a survey to be sent out. The manager is able to + configure what fields are included in the survey on this page. + """ login_url = '/login/' # redirect to login if not authenticated - def get(self, request, pk, token): - # Retrieve the questionnaire - questionnaire = get_object_or_404(Questionnaire, pk=pk) - - if not self.validate_token(token): - messages.error(request, "Invalid or expired invitation token.") - logger.error(f"Token validation failed.") - return redirect('completion_page') - - form = AnswerForm(questionnaire=questionnaire) - question_numbers = list(enumerate(questionnaire.questions.all(), start=1)) - - rating_questions = questionnaire.questions.filter(question_type='rating') - legend_text = self.get_legend_text(rating_questions) - - return render(request, 'survey/questionnaire.html', { - 'form': form, - 'questionnaire': questionnaire, - 'question_numbers': question_numbers, - 'legend_text': legend_text, - 'token': token, - }) - - def post(self, request, pk, token): - logger.info("Received POST request for questionnaire and token") - questionnaire = get_object_or_404(Questionnaire, pk=pk) - form = AnswerForm(request.POST, questionnaire=questionnaire) - - if form.is_valid(): - if self.all_agree(form, questionnaire): - self.save_answers(form, questionnaire, token) - next_questionnaire = self.get_next_questionnaire(questionnaire) - - if next_questionnaire: - logger.info(f"Redirecting to next questionnaire: {next_questionnaire.pk}") - return redirect('questionnaire', pk=next_questionnaire.pk, token=token) + def get(self, request, pk): + return self.render_survey_page(request, pk) + + def post(self, request, pk): + return self.render_survey_page(request, pk) + + + def render_survey_page(self, request, pk): + context = {} + survey = get_object_or_404(Survey, pk=pk) + + context["survey"] = survey + + return render(request, 'survey/survey.html', context) + +# TODO: Add TokenAuthenticationMixin after re-enabling the token +class SurveyResponseView(View): + """ + Participant's view of the survey. This view renders the survey configuration + allowing participant to fill in the survey form and send it for processing. + """ + + + def get(self, request: HttpRequest, pk: int, token: str): + return self.render_survey_response_page(request, pk, token, is_post=False) + + def post(self, request: HttpRequest, pk: int, token: str): + return self.render_survey_response_page(request, pk, token, is_post=True) + + + def render_survey_response_page(self, + request: HttpRequest, + pk: int, + token: str, + is_post: bool): + + survey_form_session_key = "survey_form_session" + + # Check token + + # TODO: Re-enable token once the invitation UI is in place + # if not self.validate_token(token): + # messages.error(request, "Invalid or expired invitation token.") + # logger.error(f"Token validation failed.") + # return redirect('survey_link_invalid') + + # Get the survey object and config + survey = get_object_or_404(Survey, pk=pk) + survey_config = survey.survey_config + + # TODO: Check that config is valid + + # Context for rendering + context = {} + + context["pk"] = pk + context["token"] = token + + # Gets the session data or sets it anew with section starting from 0 + session_data = {"section": 0} + if is_post: + if survey_form_session_key in request.session: + session_data = request.session[survey_form_session_key] + + current_section = session_data["section"] + survey_form_set = create_dynamic_formset(survey_config["sections"][current_section]["fields"]) + + if is_post: + # Only process if it's a post request + + # Validate current form + survey_form = survey_form_set(request.POST) + if survey_form.is_valid(): + logger.info("Form validated") + # Store form data + if "data" not in session_data: + session_data["data"] = [] + session_data["data"].append(survey_form.cleaned_data) + + current_section += 1 + if current_section < len(survey_config["sections"]): + # Go to next section + logger.info(f"Redirecting to next section") + # Store section's data + session_data["section"] = current_section + # Display the next section + survey_form = create_dynamic_formset(survey_config["sections"][current_section]["fields"]) else: - logger.info("No more questionnaires. Redirecting to completion page.") - return redirect('completion_page') + # No more sections so it's finished + logger.info("No more questions. Redirecting to completion page.") + + # Save data + SurveyResponse.objects.create(survey=survey, answers=session_data["data"]) + + # Delete session key + del request.session[survey_form_session_key] + request.session.modified = True + # TODO: Re-enable this once token has been enabled + # Invalidate token + # token = Invitation.objects.get(token=token) + # token.used = True + # token.save() + + # Go to the completion page + return redirect('completion_page') else: - messages.error(request, "You must agree to all statements to proceed.") - - context = { - 'form': form, - 'questionnaire': questionnaire, - 'question_numbers': enumerate(questionnaire.questions.all(), start=1), - 'token': token, - } - return render(request, 'survey/questionnaire.html', context=context) - - def all_agree(self, form, questionnaire): - if questionnaire.title == "Consent": - for question in questionnaire.questions.all(): - answer_text = form.cleaned_data.get(f'question_{question.id}') - if answer_text != "agree": - return False - return True - - def save_answers(self, form, questionnaire, token): - for question in questionnaire.questions.all(): - answer_text = form.cleaned_data.get(f'question_{question.id}') - Answer.objects.create( - question=question, - answer_text=answer_text, - token=token, - ) - - comment_text = form.cleaned_data.get('comments') - if comment_text: - Comment.objects.create( - text=comment_text, - questionnaire=questionnaire, - token=token, - ) - - def get_legend_text(self, rating_questions): - if rating_questions.exists(): - return "Our Organisation: (0=Not yet planned; 1=Planned; 2=Early progress; 3=Substantial Progress; 4=Established)" - return "" - - def get_next_questionnaire(self, current_questionnaire): - next_questionnaire = ( - Questionnaire.objects - .exclude(title="Consent") - .filter(pk__gt=current_questionnaire.pk) - .order_by('pk') - .first() - ) - return next_questionnaire + logger.info("Form invalid") + else: + # Return empty form if it's a get request + survey_form = survey_form_set() + + request.session[survey_form_session_key] = session_data + context["title"] = survey_config["sections"][session_data["section"]]["title"] + context["form"] = survey_form + return render(request=request, + template_name='survey/survey_response.html', + context=context) def validate_token(self, token): @@ -115,7 +154,56 @@ def validate_token(self, token): logger.warning(f"Token is invalid or expired.") return is_valid +class SurveyLinkInvalidView(View): + """ + Shown when a participant is trying to access the SurveyResponseView using an + invalid pk or token. + """ + + def get(self, request): + return render(request, "survey/survey_link_invalid_view.html" ) + class CompletionView(View): + """ + Shown when a survey is completed by a participant. + """ def get(self, request): messages.info(request, "You have completed the survey.") return render(request, 'survey/completion.html') + +class InvitationView(FormView): + + template_name = 'invitations/send_invitation.html' + form_class = InvitationForm + success_url = reverse_lazy('success_invitation') + + def form_valid(self, form): + email = form.cleaned_data['email'] + + questionnaire = Survey.objects.first() + + invitation = Invitation.objects.create(questionnaire=questionnaire) + + token = invitation.token + + # Generate the survey link with the token + survey_link = f"http://localhost:8000/survey/{questionnaire.pk}/{token}/" + + # Send the email + send_mail( + 'Your Survey Invitation', + f'Click here to start the survey: {survey_link}', + 'from@example.com', + [email], + fail_silently=False, + ) + + # Show success message + messages.success(self.request, f'Invitation sent to {email}.') + return super().form_valid(form) + + + +class SuccessInvitationView(LoginRequiredMixin, TemplateView): + + template_name = 'invitations/complete_invitation.html' \ No newline at end of file From d0fb23fe35b203d446f62b2d400331b72c0124bf Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 12:23:53 +0000 Subject: [PATCH 02/79] Add deployment script --- MANIFEST.in | 3 +++ README.md | 4 ++++ SORT/wsgi.py | 2 ++ deploy.sh | 31 ++++++++++++++++++++++++++++++ docs/deployment.md | 29 ++++++++++++++++++++++++++++ pyproject.toml | 47 ++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 ++- 7 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in create mode 100644 deploy.sh create mode 100644 docs/deployment.md create mode 100644 pyproject.toml diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..771f9d4 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +# Controlling files in the distribution +# https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html +recursive-include static * diff --git a/README.md b/README.md index 571c781..349dec7 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,7 @@ The app will be available at http://127.0.0.1:8000. ```bash python manage.py loaddata data\questionnaires.json data\questionnaires.json ``` + +# Deployment + +Please read [`docs/deployment.md`](docs/deployment.md). diff --git a/SORT/wsgi.py b/SORT/wsgi.py index 66c4f6c..6078f8e 100644 --- a/SORT/wsgi.py +++ b/SORT/wsgi.py @@ -13,4 +13,6 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "SORT.settings") +# Create WSGI application object application = get_wsgi_application() +"https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/#the-application-object" diff --git a/deploy.sh b/deploy.sh new file mode 100644 index 0000000..299f666 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +st -e + +# SORT deployment script for Ubuntu 22.04 LTS +# See: How to deploy Django +# https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/#the-application-object + +# Usage: +# Clone the repository +# git clone git@github.com:RSE-Sheffield/SORT.git +# cd SORT +# sudo bash deploy.sh + +# Options +venv_dir="/opt/sort/venv" +pip="$venv_dir/bin/pip" +python_version="python3.12" + +# Create Python virtual environment +apt install --yes -qq "$python_version" "$python_version-venv" +python3 -m venv "$venv_dir" + +# Install the SORT Django app package +$pip install . + +# Install web reverse proxy server +# Install nginx +# https://nginx.org/en/docs/install.html +apt install --yes -qq nginx + +# Configure web server diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..8d73fe9 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,29 @@ +# Deployment + +The production web server has the following architecture + +```mermaid +--- +title: SORT architecture +--- +flowchart LR +nginx --> gunicorn +gunicorn --> django +django --> postgresql +``` + + + +This app can be deployed to a web server using the script [`deploy.sh`](deploy.sh). + +Please read the following guides: + +* Django docuemntation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) +* [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) + +The relevant files are: + +* [`pyproject.toml`](pyproject.toml) defines the [Python package](https://packaging.python.org/en/latest/) +* `MANIFEST.in` lists the files that will be included in that package +* `requirements.txt` lists the dependencies for the package + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..561bbe8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,47 @@ +# Python package defnition file + +# See: Python Packaging User Guide +# https://packaging.python.org/en/latest/ + +# Django: How to write reusable apps +# https://docs.djangoproject.com/en/5.1/intro/reusable-apps/ + +# Related files: +# - requirements.txt lists the dependencies +# - MANIFEST.in includes or excludes other files from the package +# - README.md describes the package + +[build-system] +# https://setuptools.pypa.io/en/latest/index.html +requires = ["setuptools>=75"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["SORT"] + +[project] +name = "SORT" +version = "0.0.1" +readme = "README.md" +# https://devguide.python.org/versions/ +requires-python = ">= 3.10" +authors = [ + { name = "Farhad Allian", email = "farhad.allian@sheffield.ac.uk" }, + { name = "Twin Karmakharm", email = "t.karmakharm@sheffield.ac.uk" }, + { name = "Joe Heffer", email = "j.heffer@sheffield.ac.uk" }, +] +classifiers = [ + "Environment :: Web Environment", + "Framework :: Django", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", +] +# Get dependencies from requirements.txt +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } diff --git a/requirements.txt b/requirements.txt index 47d408e..e57f04d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,5 @@ traitlets==5.14.3 typing_extensions==4.12.2 tzdata==2024.1 wcwidth==0.2.13 -django_debug_toolbar==4.4.6 \ No newline at end of file +django_debug_toolbar==4.4.6 +gunicorn==23.* From aa78a03c32a6ed65209e0f8745b9c7eadabf8ad4 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 12:26:59 +0000 Subject: [PATCH 03/79] Fix syntax error --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 299f666..b5e1f19 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -st -e +set -e # SORT deployment script for Ubuntu 22.04 LTS # See: How to deploy Django From 0d7fd29ecec8389887f1b1c286eb5d9acfdadcea Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 12:49:05 +0000 Subject: [PATCH 04/79] Add shell and systemd linting --- .github/workflows/lint-shell-scripts.yaml | 14 ++++++++++++++ .github/workflows/lint-systemd.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .github/workflows/lint-shell-scripts.yaml create mode 100644 .github/workflows/lint-systemd.yaml diff --git a/.github/workflows/lint-shell-scripts.yaml b/.github/workflows/lint-shell-scripts.yaml new file mode 100644 index 0000000..f09e8b8 --- /dev/null +++ b/.github/workflows/lint-shell-scripts.yaml @@ -0,0 +1,14 @@ +# Lint shell scripts +# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions +# https://github.com/marketplace/actions/shell-linter +name: Lint shell scripts +on: push +jobs: + lint_shell: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: azohra/shell-linter@v0.6.0 + with: + severity: 'warning' + exclude-paths: 'LICENSE' diff --git a/.github/workflows/lint-systemd.yaml b/.github/workflows/lint-systemd.yaml new file mode 100644 index 0000000..73db1d0 --- /dev/null +++ b/.github/workflows/lint-systemd.yaml @@ -0,0 +1,14 @@ +# GitHub Actions workflow for linting the systemd unit files +# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions +name: Lint systemd units +on: [ push ] +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install systemdlint + run: pip install systemdlint==1.* + - name: Lint systemd units + run: systemdlint ./config/systemd/* From 4476fd65b900a79e8ea6069a02cab4cdfc6d334f Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 13:12:03 +0000 Subject: [PATCH 05/79] Add gunicorn systemd service --- config/README.md | 3 +++ config/systemd/gunicorn.service | 30 ++++++++++++++++++++++++++++++ config/systemd/gunicorn.socket | 17 +++++++++++++++++ deploy.sh | 5 +++++ docs/deployment.md | 11 +++++++---- pyproject.toml | 7 +++---- 6 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 config/README.md create mode 100644 config/systemd/gunicorn.service create mode 100644 config/systemd/gunicorn.socket diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..0060b16 --- /dev/null +++ b/config/README.md @@ -0,0 +1,3 @@ +# Server configuration + +This directory contains configuration files for the production web server. diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service new file mode 100644 index 0000000..c7a35f3 --- /dev/null +++ b/config/systemd/gunicorn.service @@ -0,0 +1,30 @@ +# This is a systemd unit that defines the Gunicorn service. +# https://docs.gunicorn.org/en/stable/deploy.html#systemd +# https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html + +[Unit] +Description=gunicorn service +Requires=gunicorn.socket +After=network.target + +[Service] +# gunicorn can let systemd know when it is ready +Type=notify +NotifyAccess=main +# the specific user that our service will run as +User=gunicorn +Group=gunicorn +# this user can be transiently created by systemd +DynamicUser=true +RuntimeDirectory=gunicorn +WorkingDirectory=/opt/sort/venv/lib/python3.12/site-packages/SORT +ExecStart=/opt/sort/venv/bin/gunicorn sort.wsgi +ExecReload=/bin/kill -s HUP $MAINPID +KillMode=mixed +TimeoutStopSec=5 +PrivateTmp=true +# if your app does not need administrative capabilities, let systemd know +ProtectSystem=strict + +[Install] +WantedBy=multi-user.target diff --git a/config/systemd/gunicorn.socket b/config/systemd/gunicorn.socket new file mode 100644 index 0000000..3bc872e --- /dev/null +++ b/config/systemd/gunicorn.socket @@ -0,0 +1,17 @@ +# https://docs.gunicorn.org/en/stable/deploy.html#systemd +# https://www.freedesktop.org/software/systemd/man/latest/systemd.socket.html +[Unit] +Description=gunicorn socket + +[Socket] +ListenStream=/run/gunicorn.sock +# Our service won't need permissions for the socket, since it +# inherits the file descriptor by socket activation. +# Only the nginx daemon will need access to the socket: +SocketUser=www-data +SocketGroup=www-data +# Once the user/group is correct, restrict the permissions: +SocketMode=0660 + +[Install] +WantedBy=sockets.target diff --git a/deploy.sh b/deploy.sh index b5e1f19..e4c6042 100644 --- a/deploy.sh +++ b/deploy.sh @@ -17,15 +17,20 @@ pip="$venv_dir/bin/pip" python_version="python3.12" # Create Python virtual environment +apt update apt install --yes -qq "$python_version" "$python_version-venv" python3 -m venv "$venv_dir" # Install the SORT Django app package $pip install . +# Install Gunicorn service +cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service + # Install web reverse proxy server # Install nginx # https://nginx.org/en/docs/install.html apt install --yes -qq nginx +rm -f /etc/nginx/sites-enabled/default # Configure web server diff --git a/docs/deployment.md b/docs/deployment.md index 8d73fe9..e1baf82 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -7,9 +7,12 @@ The production web server has the following architecture title: SORT architecture --- flowchart LR -nginx --> gunicorn -gunicorn --> django -django --> postgresql +Browser -- "HTTPS Port 443" --> nginx +subgraph UoS +nginx -- "Unix socket" --> Gunicorn +Gunicorn -- "WSGI" --> Django +Django --> PostgreSQL +end ``` @@ -18,7 +21,7 @@ This app can be deployed to a web server using the script [`deploy.sh`](deploy.s Please read the following guides: -* Django docuemntation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) +* Django documentation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) * [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) The relevant files are: diff --git a/pyproject.toml b/pyproject.toml index 561bbe8..4288894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,12 +12,11 @@ # - README.md describes the package [build-system] -# https://setuptools.pypa.io/en/latest/index.html +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html requires = ["setuptools>=75"] build-backend = "setuptools.build_meta" - -[tool.setuptools] -packages = ["SORT"] +[tool.setuptools.packages.find] +where = ["."] [project] name = "SORT" From d75a0f426b9bd58a240fb79ede2ca3ffd65eeaa3 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 13:31:40 +0000 Subject: [PATCH 06/79] Don't use pyproject.toml It's a can of worms for a Django app --- deploy.sh | 6 ++++-- pyproject.toml | 46 ---------------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) delete mode 100644 pyproject.toml diff --git a/deploy.sh b/deploy.sh index e4c6042..decf02e 100644 --- a/deploy.sh +++ b/deploy.sh @@ -12,7 +12,8 @@ set -e # sudo bash deploy.sh # Options -venv_dir="/opt/sort/venv" +sort_dir="/opt/sort" +venv_dir="$sort_dir/venv" pip="$venv_dir/bin/pip" python_version="python3.12" @@ -22,7 +23,8 @@ apt install --yes -qq "$python_version" "$python_version-venv" python3 -m venv "$venv_dir" # Install the SORT Django app package -$pip install . +$pip install -r requirements.txt +cp --recursive * "$sort_dir/" # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 4288894..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,46 +0,0 @@ -# Python package defnition file - -# See: Python Packaging User Guide -# https://packaging.python.org/en/latest/ - -# Django: How to write reusable apps -# https://docs.djangoproject.com/en/5.1/intro/reusable-apps/ - -# Related files: -# - requirements.txt lists the dependencies -# - MANIFEST.in includes or excludes other files from the package -# - README.md describes the package - -[build-system] -# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html -requires = ["setuptools>=75"] -build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -where = ["."] - -[project] -name = "SORT" -version = "0.0.1" -readme = "README.md" -# https://devguide.python.org/versions/ -requires-python = ">= 3.10" -authors = [ - { name = "Farhad Allian", email = "farhad.allian@sheffield.ac.uk" }, - { name = "Twin Karmakharm", email = "t.karmakharm@sheffield.ac.uk" }, - { name = "Joe Heffer", email = "j.heffer@sheffield.ac.uk" }, -] -classifiers = [ - "Environment :: Web Environment", - "Framework :: Django", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Internet :: WWW/HTTP :: Dynamic Content", -] -# Get dependencies from requirements.txt -dynamic = ["dependencies"] - -[tool.setuptools.dynamic] -dependencies = { file = ["requirements.txt"] } From a84d3f9fbba1e83bfc7396493a5331de63158f00 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 13:33:31 +0000 Subject: [PATCH 07/79] Install Gunicorn socket --- deploy.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy.sh b/deploy.sh index decf02e..8594790 100644 --- a/deploy.sh +++ b/deploy.sh @@ -9,7 +9,7 @@ set -e # Clone the repository # git clone git@github.com:RSE-Sheffield/SORT.git # cd SORT -# sudo bash deploy.sh +# sudo bash -x deploy.sh # Options sort_dir="/opt/sort" @@ -18,7 +18,7 @@ pip="$venv_dir/bin/pip" python_version="python3.12" # Create Python virtual environment -apt update +apt update -qq apt install --yes -qq "$python_version" "$python_version-venv" python3 -m venv "$venv_dir" @@ -28,6 +28,7 @@ cp --recursive * "$sort_dir/" # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service +cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket # Install web reverse proxy server # Install nginx From df444865908bf8c12123d63b6a490b052940d4df Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 13:39:19 +0000 Subject: [PATCH 08/79] - --- config/systemd/gunicorn.service | 2 +- deploy.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index c7a35f3..c74c33b 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -17,7 +17,7 @@ Group=gunicorn # this user can be transiently created by systemd DynamicUser=true RuntimeDirectory=gunicorn -WorkingDirectory=/opt/sort/venv/lib/python3.12/site-packages/SORT +WorkingDirectory=/opt/sort ExecStart=/opt/sort/venv/bin/gunicorn sort.wsgi ExecReload=/bin/kill -s HUP $MAINPID KillMode=mixed diff --git a/deploy.sh b/deploy.sh index 8594790..fc54712 100644 --- a/deploy.sh +++ b/deploy.sh @@ -30,6 +30,9 @@ cp --recursive * "$sort_dir/" cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket +systemctl enable gunicorn.service +systemctl enable gunicorn.socket + # Install web reverse proxy server # Install nginx # https://nginx.org/en/docs/install.html From 249c5d83e3ed13229751a8f3085bda1a27e57114 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 13:43:16 +0000 Subject: [PATCH 09/79] Fix SORT module name --- config/systemd/gunicorn.service | 2 +- deploy.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index c74c33b..2722320 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -18,7 +18,7 @@ Group=gunicorn DynamicUser=true RuntimeDirectory=gunicorn WorkingDirectory=/opt/sort -ExecStart=/opt/sort/venv/bin/gunicorn sort.wsgi +ExecStart=/opt/sort/venv/bin/gunicorn SORT.wsgi ExecReload=/bin/kill -s HUP $MAINPID KillMode=mixed TimeoutStopSec=5 diff --git a/deploy.sh b/deploy.sh index fc54712..cb16d71 100644 --- a/deploy.sh +++ b/deploy.sh @@ -29,6 +29,7 @@ cp --recursive * "$sort_dir/" # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket +systemctl daemon-reload systemctl enable gunicorn.service systemctl enable gunicorn.socket From 3ac712fddf0f1abe50964d92a026d01fdddeab1e Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:02:16 +0000 Subject: [PATCH 10/79] Configure NGINX proxy pass --- .github/workflows/lint-nginx.yaml | 14 ++++++++++++ config/nginx/gunicorn.conf | 38 +++++++++++++++++++++++++++++++ config/systemd/gunicorn.service | 1 + config/systemd/gunicorn.socket | 2 +- deploy.sh | 5 ++-- docs/deployment.md | 29 +++++++++++++++++++++-- 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/lint-nginx.yaml create mode 100644 config/nginx/gunicorn.conf diff --git a/.github/workflows/lint-nginx.yaml b/.github/workflows/lint-nginx.yaml new file mode 100644 index 0000000..1a98c5b --- /dev/null +++ b/.github/workflows/lint-nginx.yaml @@ -0,0 +1,14 @@ +# GitHub Actions workflow for validating NGINX configuration files +# https://github.com/jhinch/nginx-linter +name: Lint NGINX config files +on: [ push ] +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install nginx-linter + run: npm install -g nginx-linter + - name: Lint systemd units + run: nginx-linter diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf new file mode 100644 index 0000000..6f4b1b9 --- /dev/null +++ b/config/nginx/gunicorn.conf @@ -0,0 +1,38 @@ +# https://docs.gunicorn.org/en/stable/deploy.html#nginx-configuration +upstream app_server { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + # for UNIX domain socket setups + server unix:/run/gunicorn/gunicorn.sock fail_timeout=0; +} + +server { + # if no Host match, close the connection to prevent host spoofing + listen 80 default_server; + return 444; +} + +server { + # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen + listen 80 deferred; + client_max_body_size 4G; + server_name sort-web-dev.shef.ac.uk www.sort-web-dev.shef.ac.uk; + keepalive_timeout 5; + # path for static files + root /opt/sort/static/; + + location / { + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://app_server; + } +} diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index 2722320..4483ef9 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -25,6 +25,7 @@ TimeoutStopSec=5 PrivateTmp=true # if your app does not need administrative capabilities, let systemd know ProtectSystem=strict +Delegate=yes [Install] WantedBy=multi-user.target diff --git a/config/systemd/gunicorn.socket b/config/systemd/gunicorn.socket index 3bc872e..34e7342 100644 --- a/config/systemd/gunicorn.socket +++ b/config/systemd/gunicorn.socket @@ -4,7 +4,7 @@ Description=gunicorn socket [Socket] -ListenStream=/run/gunicorn.sock +ListenStream=/run/gunicorn/gunicorn.sock # Our service won't need permissions for the socket, since it # inherits the file descriptor by socket activation. # Only the nginx daemon will need access to the socket: diff --git a/deploy.sh b/deploy.sh index cb16d71..fef1092 100644 --- a/deploy.sh +++ b/deploy.sh @@ -30,7 +30,6 @@ cp --recursive * "$sort_dir/" cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket systemctl daemon-reload - systemctl enable gunicorn.service systemctl enable gunicorn.socket @@ -38,6 +37,8 @@ systemctl enable gunicorn.socket # Install nginx # https://nginx.org/en/docs/install.html apt install --yes -qq nginx -rm -f /etc/nginx/sites-enabled/default # Configure web server +rm -f /etc/nginx/sites-enabled/default +cp config/nginx/*.conf /etc/nginx/sites-available +ln -s /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf diff --git a/docs/deployment.md b/docs/deployment.md index e1baf82..49db773 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,13 +1,13 @@ # Deployment -The production web server has the following architecture +The production web server has the following architecture: ```mermaid --- title: SORT architecture --- flowchart LR -Browser -- "HTTPS Port 443" --> nginx +Browser -- "HTTPS port 443" --> nginx subgraph UoS nginx -- "Unix socket" --> Gunicorn Gunicorn -- "WSGI" --> Django @@ -30,3 +30,28 @@ The relevant files are: * `MANIFEST.in` lists the files that will be included in that package * `requirements.txt` lists the dependencies for the package +# Monitoring + +## View service status + +```bash +sudo systemctl status gunicorn +sudo systemctl status nginx +``` + +# Control + +The services are controlled using [`systemd`](https://systemd.io/), which is the service management system on Ubuntu 24. To launch services: + +```bash +sudo systemctl start gunicorn +sudo systemctl start nginx +``` + +To stop services: + +```bash +sudo systemctl stop gunicorn +sudo systemctl stop nginx +``` + From e2ca956359129680463593a22906d658d197ba2b Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:06:52 +0000 Subject: [PATCH 11/79] - --- .github/workflows/lint-nginx.yaml | 4 ++-- MANIFEST.in | 3 --- docs/deployment.md | 8 +++----- 3 files changed, 5 insertions(+), 10 deletions(-) delete mode 100644 MANIFEST.in diff --git a/.github/workflows/lint-nginx.yaml b/.github/workflows/lint-nginx.yaml index 1a98c5b..b97631e 100644 --- a/.github/workflows/lint-nginx.yaml +++ b/.github/workflows/lint-nginx.yaml @@ -10,5 +10,5 @@ jobs: uses: actions/checkout@v4 - name: Install nginx-linter run: npm install -g nginx-linter - - name: Lint systemd units - run: nginx-linter + - name: Run nginx linter + run: nginx-linter --help diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 771f9d4..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -# Controlling files in the distribution -# https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html -recursive-include static * diff --git a/docs/deployment.md b/docs/deployment.md index 49db773..d866413 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -17,18 +17,16 @@ end -This app can be deployed to a web server using the script [`deploy.sh`](deploy.sh). +This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh). -Please read the following guides: +You may also refer to the following guides: * Django documentation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) * [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) The relevant files are: -* [`pyproject.toml`](pyproject.toml) defines the [Python package](https://packaging.python.org/en/latest/) -* `MANIFEST.in` lists the files that will be included in that package -* `requirements.txt` lists the dependencies for the package +* The `config/` directory contains server configuration files. # Monitoring From d363d54988b5da74dc9b25eeb5b63e4d8c06ab41 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:08:38 +0000 Subject: [PATCH 12/79] Fix nginx linter path --- .github/workflows/lint-nginx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-nginx.yaml b/.github/workflows/lint-nginx.yaml index b97631e..4f4e974 100644 --- a/.github/workflows/lint-nginx.yaml +++ b/.github/workflows/lint-nginx.yaml @@ -11,4 +11,4 @@ jobs: - name: Install nginx-linter run: npm install -g nginx-linter - name: Run nginx linter - run: nginx-linter --help + run: nginx-linter --include config/nginx/* --no-follow-includes From 83cb30f873c30de375c51cc634e55fd5786d6342 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:11:12 +0000 Subject: [PATCH 13/79] Fix indent --- config/nginx/gunicorn.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 6f4b1b9..1db1949 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -22,17 +22,17 @@ server { root /opt/sort/static/; location / { - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; } location @proxy_to_app { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - # we don't want nginx trying to do something clever with - # redirects, we set the Host: header above already. - proxy_redirect off; - proxy_pass http://app_server; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + proxy_pass http://app_server; } } From 4398da41b77ddd3ac98432cdc5a53ef318afd33e Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:26:53 +0000 Subject: [PATCH 14/79] Reload services on deploy --- deploy.sh | 2 ++ docs/deployment.md | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/deploy.sh b/deploy.sh index fef1092..a74de25 100644 --- a/deploy.sh +++ b/deploy.sh @@ -32,6 +32,7 @@ cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket systemctl daemon-reload systemctl enable gunicorn.service systemctl enable gunicorn.socket +systemctl reload gunicorn.service # Install web reverse proxy server # Install nginx @@ -42,3 +43,4 @@ apt install --yes -qq nginx rm -f /etc/nginx/sites-enabled/default cp config/nginx/*.conf /etc/nginx/sites-available ln -s /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf +systemctl reload nginx.service diff --git a/docs/deployment.md b/docs/deployment.md index d866413..1bdaa95 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -37,6 +37,20 @@ sudo systemctl status gunicorn sudo systemctl status nginx ``` +# View logs + +[nginx logs](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/) + +```bash +sudo tail /var/log/nginx/error.log +``` + +[Gunicorn logs](https://docs.gunicorn.org/en/stable/settings.html#logging) + +```bash + sudo journalctl -u gunicorn.service +``` + # Control The services are controlled using [`systemd`](https://systemd.io/), which is the service management system on Ubuntu 24. To launch services: From 6251386aa82b2b410d9564b46a0ad6e4213fbd7f Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:30:06 +0000 Subject: [PATCH 15/79] Revert Gunicorn socket path --- config/nginx/gunicorn.conf | 2 +- config/systemd/gunicorn.socket | 2 +- deploy.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 1db1949..0818575 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -3,7 +3,7 @@ upstream app_server { # fail_timeout=0 means we always retry an upstream even if it failed # to return a good HTTP response # for UNIX domain socket setups - server unix:/run/gunicorn/gunicorn.sock fail_timeout=0; + server unix:/run/gunicorn.sock fail_timeout=0; } server { diff --git a/config/systemd/gunicorn.socket b/config/systemd/gunicorn.socket index 34e7342..3bc872e 100644 --- a/config/systemd/gunicorn.socket +++ b/config/systemd/gunicorn.socket @@ -4,7 +4,7 @@ Description=gunicorn socket [Socket] -ListenStream=/run/gunicorn/gunicorn.sock +ListenStream=/run/gunicorn.sock # Our service won't need permissions for the socket, since it # inherits the file descriptor by socket activation. # Only the nginx daemon will need access to the socket: diff --git a/deploy.sh b/deploy.sh index a74de25..44887ab 100644 --- a/deploy.sh +++ b/deploy.sh @@ -42,5 +42,6 @@ apt install --yes -qq nginx # Configure web server rm -f /etc/nginx/sites-enabled/default cp config/nginx/*.conf /etc/nginx/sites-available -ln -s /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf +# Enable the site by creating a symbolic link +ln --symbolic --force /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf systemctl reload nginx.service From b2dc7cbd99d533a909bd0af06ad8fd3f68ffd732 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:49:00 +0000 Subject: [PATCH 16/79] Add ALLOWED_HOSTS env var --- SORT/settings.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 1b5ee8a..f28d01e 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -19,7 +19,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ @@ -29,8 +28,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] - +ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '').split() # Application definition @@ -84,7 +82,6 @@ WSGI_APPLICATION = "SORT.wsgi.application" - # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases @@ -118,7 +115,6 @@ }, ] - # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ @@ -130,7 +126,6 @@ USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ @@ -138,8 +133,6 @@ STATICFILES_DIRS = [BASE_DIR / 'static'] - - # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -148,14 +141,13 @@ LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/" - # FA: End session when the browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = True # FA: 30 minutes before automatic log out SESSION_COOKIE_AGE = 1800 -PASSWORD_RESET_TIMEOUT = 1800 # FA: default to expire after 30 minutes +PASSWORD_RESET_TIMEOUT = 1800 # FA: default to expire after 30 minutes # FA: for local testing emails: @@ -174,5 +166,4 @@ # FA: for production: -#EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" - +# EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" From 2e452928b9c903750c04d550821c31f41ea90e29 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:54:28 +0000 Subject: [PATCH 17/79] Add deployment auto check workflow --- .github/workflows/check-django.yaml | 11 +++++++++++ deploy.sh | 2 +- docs/deployment.md | 21 +++++++++++++++++++-- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/check-django.yaml diff --git a/.github/workflows/check-django.yaml b/.github/workflows/check-django.yaml new file mode 100644 index 0000000..0c70fa6 --- /dev/null +++ b/.github/workflows/check-django.yaml @@ -0,0 +1,11 @@ +# Django deployment checks +# https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ +name: Check Django deployment +on: [ push ] +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - run: python manage.py check --deploy diff --git a/deploy.sh b/deploy.sh index 44887ab..bcd0533 100644 --- a/deploy.sh +++ b/deploy.sh @@ -23,7 +23,7 @@ apt install --yes -qq "$python_version" "$python_version-venv" python3 -m venv "$venv_dir" # Install the SORT Django app package -$pip install -r requirements.txt +$pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" # Install Gunicorn service diff --git a/docs/deployment.md b/docs/deployment.md index 1bdaa95..0cea965 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,5 +1,7 @@ # Deployment +This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh) and configured as described in the section below. + The production web server has the following architecture: ```mermaid @@ -17,8 +19,6 @@ end -This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh). - You may also refer to the following guides: * Django documentation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) @@ -28,6 +28,22 @@ The relevant files are: * The `config/` directory contains server configuration files. +# Configuration + +To configure the environment variables for the service: + +```bash +sudo systemctl edit gunicorn.service +``` + +Add the following lines: + +```ini +[Service] +Environment="DJANGO_SECRET_KEY=*********" +Environment="DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk www.sort-web-app.shef.ac.uk" +``` + # Monitoring ## View service status @@ -43,6 +59,7 @@ sudo systemctl status nginx ```bash sudo tail /var/log/nginx/error.log +sudo tail /var/log/nginx/access.log ``` [Gunicorn logs](https://docs.gunicorn.org/en/stable/settings.html#logging) From 7dbc1f7bd2f4383add27245bbac6034b6b719f86 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:55:42 +0000 Subject: [PATCH 18/79] Install Django package --- .github/workflows/check-django.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-django.yaml b/.github/workflows/check-django.yaml index 0c70fa6..ca89530 100644 --- a/.github/workflows/check-django.yaml +++ b/.github/workflows/check-django.yaml @@ -8,4 +8,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - run: python manage.py check --deploy + - name: Install Django + run: pip install django + - name: Run Django deployment checks + run: python manage.py check --deploy From 3cac085510c8ced90e3e5f822bf212e550cd2200 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 14:58:41 +0000 Subject: [PATCH 19/79] Install dotenv --- .github/workflows/check-django.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-django.yaml b/.github/workflows/check-django.yaml index ca89530..e7aa161 100644 --- a/.github/workflows/check-django.yaml +++ b/.github/workflows/check-django.yaml @@ -9,6 +9,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Install Django - run: pip install django + run: pip install django dotenv - name: Run Django deployment checks run: python manage.py check --deploy From 9f874792aace0ba28e0a6c0292cfe947efcb148b Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 15:08:47 +0000 Subject: [PATCH 20/79] Serve static files https://www.digitalocean.com/community/tutorials/how-to-set-up-django-with-postgres-nginx-and-gunicorn-on-ubuntu --- .github/workflows/check-django.yaml | 14 -------------- config/nginx/gunicorn.conf | 13 ++++++------- 2 files changed, 6 insertions(+), 21 deletions(-) delete mode 100644 .github/workflows/check-django.yaml diff --git a/.github/workflows/check-django.yaml b/.github/workflows/check-django.yaml deleted file mode 100644 index e7aa161..0000000 --- a/.github/workflows/check-django.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Django deployment checks -# https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ -name: Check Django deployment -on: [ push ] -jobs: - lint: - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Django - run: pip install django dotenv - - name: Run Django deployment checks - run: python manage.py check --deploy diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 0818575..b290a8d 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -16,17 +16,16 @@ server { # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen listen 80 deferred; client_max_body_size 4G; - server_name sort-web-dev.shef.ac.uk www.sort-web-dev.shef.ac.uk; + server_name sort-web-app.shef.ac.uk www.sort-web-app.shef.ac.uk; keepalive_timeout 5; - # path for static files - root /opt/sort/static/; - location / { - # checks for static file, if not found proxy to app - try_files $uri @proxy_to_app; + # Serve static files without invoking Python WSGI + location /static/ { + root /opt/sort/static; } - location @proxy_to_app { + # Proxy forward to the WSGI Python app + location / { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $http_host; From bcbfeebef7b0f50edef74c1914088ddac2ade8dd Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 15:16:31 +0000 Subject: [PATCH 21/79] Remove spoof check --- config/nginx/gunicorn.conf | 6 ------ docs/deployment.md | 24 +++++++++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index b290a8d..edf7313 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -6,12 +6,6 @@ upstream app_server { server unix:/run/gunicorn.sock fail_timeout=0; } -server { - # if no Host match, close the connection to prevent host spoofing - listen 80 default_server; - return 444; -} - server { # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen listen 80 deferred; diff --git a/docs/deployment.md b/docs/deployment.md index 0cea965..b76defb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -10,14 +10,20 @@ title: SORT architecture --- flowchart LR Browser -- "HTTPS port 443" --> nginx -subgraph UoS -nginx -- "Unix socket" --> Gunicorn -Gunicorn -- "WSGI" --> Django -Django --> PostgreSQL +subgraph University of Sheffield network + subgraph sort-web-app machine + nginx -- "Unix socket" --> Gunicorn + Gunicorn -- "WSGI" --> Django + Django --> PostgreSQL + end end ``` +When accessing the web site, the following process happens: +1. A user uses their web browser to access the server using the HTTPS port 443; +2. The request is sent to the web server and is handled by Nginx; +3. You may also refer to the following guides: @@ -55,17 +61,17 @@ sudo systemctl status nginx # View logs -[nginx logs](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/) +View [nginx logs](https://docs.nginx.com/nginx/admin-guide/monitoring/logging/) ```bash -sudo tail /var/log/nginx/error.log -sudo tail /var/log/nginx/access.log +sudo tail --follow /var/log/nginx/access.log +sudo tail --follow /var/log/nginx/error.log ``` -[Gunicorn logs](https://docs.gunicorn.org/en/stable/settings.html#logging) +View [Gunicorn logs](https://docs.gunicorn.org/en/stable/settings.html#logging) ```bash - sudo journalctl -u gunicorn.service + sudo journalctl -u gunicorn.service --follow ``` # Control From 3479a76f12c352cca9acc1775a71f1e9de8d53d0 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 15:36:58 +0000 Subject: [PATCH 22/79] Collect static files https://docs.djangoproject.com/en/5.1/howto/static-files/#deployment --- SORT/settings.py | 2 ++ config/nginx/gunicorn.conf | 2 +- deploy.sh | 2 ++ docs/deployment.md | 12 +++++++++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index f28d01e..1027a34 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -167,3 +167,5 @@ # FA: for production: # EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" + +STATIC_ROOT = os.getenv('DJANGO_STATIC_ROOT') diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index edf7313..bfc0ef1 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -15,7 +15,7 @@ server { # Serve static files without invoking Python WSGI location /static/ { - root /opt/sort/static; + root /var/www/sort; } # Proxy forward to the WSGI Python app diff --git a/deploy.sh b/deploy.sh index bcd0533..1bcfb22 100644 --- a/deploy.sh +++ b/deploy.sh @@ -25,6 +25,8 @@ python3 -m venv "$venv_dir" # Install the SORT Django app package $pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" +# Collect static files +"$venv_dir/bin/python" manage.py collectstatic # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service diff --git a/docs/deployment.md b/docs/deployment.md index b76defb..539a8d5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -36,7 +36,17 @@ The relevant files are: # Configuration -To configure the environment variables for the service: +To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service. + +To edit the environment file: + +```bash +sudo nano /opt/sort/.env +``` + +## Service options + +To edit the system service options: ```bash sudo systemctl edit gunicorn.service From bc9ecc82fa38668d6902b73acd89ea12fba9e8d4 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 15:45:16 +0000 Subject: [PATCH 23/79] Collect static files on service start --- config/systemd/gunicorn.service | 3 +++ deploy.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index 4483ef9..6d116ce 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -18,6 +18,9 @@ Group=gunicorn DynamicUser=true RuntimeDirectory=gunicorn WorkingDirectory=/opt/sort +# Collect static files on launch +# See: https://docs.djangoproject.com/en/5.1/howto/static-files/#deployment +ExecStartPre=/opt/sort/venv/bin/python manage.py collectstatic --no-input ExecStart=/opt/sort/venv/bin/gunicorn SORT.wsgi ExecReload=/bin/kill -s HUP $MAINPID KillMode=mixed diff --git a/deploy.sh b/deploy.sh index 1bcfb22..3eb03bc 100644 --- a/deploy.sh +++ b/deploy.sh @@ -16,6 +16,7 @@ sort_dir="/opt/sort" venv_dir="$sort_dir/venv" pip="$venv_dir/bin/pip" python_version="python3.12" +python="$venv_dir/bin/python" # Create Python virtual environment apt update -qq @@ -25,8 +26,6 @@ python3 -m venv "$venv_dir" # Install the SORT Django app package $pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" -# Collect static files -"$venv_dir/bin/python" manage.py collectstatic # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service @@ -35,6 +34,7 @@ systemctl daemon-reload systemctl enable gunicorn.service systemctl enable gunicorn.socket systemctl reload gunicorn.service +systemctl restart gunicorn.service # Install web reverse proxy server # Install nginx From b4b7099f5d5dd12d91bf7d834b586985c91c9ade Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 15:49:18 +0000 Subject: [PATCH 24/79] Install static files in deployment script --- config/systemd/gunicorn.service | 3 --- deploy.sh | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index 6d116ce..4483ef9 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -18,9 +18,6 @@ Group=gunicorn DynamicUser=true RuntimeDirectory=gunicorn WorkingDirectory=/opt/sort -# Collect static files on launch -# See: https://docs.djangoproject.com/en/5.1/howto/static-files/#deployment -ExecStartPre=/opt/sort/venv/bin/python manage.py collectstatic --no-input ExecStart=/opt/sort/venv/bin/gunicorn SORT.wsgi ExecReload=/bin/kill -s HUP $MAINPID KillMode=mixed diff --git a/deploy.sh b/deploy.sh index 3eb03bc..d7531cb 100644 --- a/deploy.sh +++ b/deploy.sh @@ -27,6 +27,9 @@ python3 -m venv "$venv_dir" $pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" +# Install static files +(cd "$sort_dir" && exec $python manage.py collectstatic --no-input) + # Install Gunicorn service cp --verbose config/systemd/gunicorn.service /etc/systemd/system/gunicorn.service cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket @@ -34,7 +37,6 @@ systemctl daemon-reload systemctl enable gunicorn.service systemctl enable gunicorn.socket systemctl reload gunicorn.service -systemctl restart gunicorn.service # Install web reverse proxy server # Install nginx From a557a3ef3d5d51a40615afe3e4dd1fb853e31b94 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Wed, 8 Jan 2025 17:01:58 +0000 Subject: [PATCH 25/79] Start working on PostgreSQL installation --- SORT/settings.py | 7 +--- config/nginx/gunicorn.conf | 1 + deploy.sh | 4 ++ docs/deployment.md | 76 ++++++++++++++++++++++++++++++++------ 4 files changed, 71 insertions(+), 17 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 1027a34..5c35469 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -88,12 +88,7 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / 'db.sqlite3', - # "NAME": os.getenv("DB_NAME"), - # "USER": os.getenv("DB_USER"), - # "PASSWORD": os.getenv("DB_PASSWORD"), - # "HOST": os.getenv("DB_HOST"), - # "PORT": os.getenv("DB_PORT"), + "NAME": BASE_DIR / 'db.sqlite3' } } diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index bfc0ef1..089e3c8 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -15,6 +15,7 @@ server { # Serve static files without invoking Python WSGI location /static/ { + # https://nginx.org/en/docs/http/ngx_http_core_module.html#root root /var/www/sort; } diff --git a/deploy.sh b/deploy.sh index d7531cb..77faeea 100644 --- a/deploy.sh +++ b/deploy.sh @@ -49,3 +49,7 @@ cp config/nginx/*.conf /etc/nginx/sites-available # Enable the site by creating a symbolic link ln --symbolic --force /etc/nginx/sites-available/gunicorn.conf /etc/nginx/sites-enabled/gunicorn.conf systemctl reload nginx.service + +# Install PostgreSQL database +# https://ubuntu.com/server/docs/install-and-configure-postgresql +apt install --yes -qq postgresql diff --git a/docs/deployment.md b/docs/deployment.md index 539a8d5..e4a85a0 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,7 +1,5 @@ # Deployment -This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh) and configured as described in the section below. - The production web server has the following architecture: ```mermaid @@ -13,6 +11,7 @@ Browser -- "HTTPS port 443" --> nginx subgraph University of Sheffield network subgraph sort-web-app machine nginx -- "Unix socket" --> Gunicorn + nginx --> static["Static files"] Gunicorn -- "WSGI" --> Django Django --> PostgreSQL end @@ -23,7 +22,8 @@ When accessing the web site, the following process happens: 1. A user uses their web browser to access the server using the HTTPS port 443; 2. The request is sent to the web server and is handled by Nginx; -3. +3. Nginx uses the web server gateway interface (WSGI) to access the Django application; +4. Django uses the PostgreSQL database to store data. You may also refer to the following guides: @@ -34,6 +34,14 @@ The relevant files are: * The `config/` directory contains server configuration files. +# Deployment process + +This app can be deployed to a web server using the script [`deploy.sh`](../deploy.sh) and configured as described in the section below. + +1. Configure the `.env` file as described below. +2. Run the deployment script: `sudo bash -x deploy.sh` +3. Configure the database + # Configuration To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service. @@ -41,25 +49,70 @@ To configure the environment variables for the service, you can either edit the To edit the environment file: ```bash +sudo mkdir --parents /opt/sort sudo nano /opt/sort/.env ``` -## Service options +This file would typically look similar to this: + +```ini +DJANGO_SECRET_KEY=******** +DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk +DJANGO_STATIC_ROOT=/var/www/sort/static +``` -To edit the system service options: +# Database installation + +To run these commands, switch to the `postgres` user: ```bash -sudo systemctl edit gunicorn.service +sudo su - postgres ``` -Add the following lines: +## Create a database -```ini -[Service] -Environment="DJANGO_SECRET_KEY=*********" -Environment="DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk www.sort-web-app.shef.ac.uk" +[Create a database](https://www.postgresql.org/docs/16/tutorial-createdb.html) + +```bash +createdb sort +``` + +## Create a user + +The app needs credentials to access the database. + +Create a user: + +```bash +createuser sort ``` +# Management + +To use the Django management tool + +```bash +sort_dir="/opt/sort" +venv_dir="$sort_dir/venv" +python="$venv_dir/bin/python" +cd "$sort_dir" +$python "$sort_dir"/manage.py help +``` + +Migrate the database + +```bash +sudo $python manage.py migrate +``` + +Create a super-user + +```bash +sudo $python manage.py createsuperuser +``` + + + # Monitoring ## View service status @@ -67,6 +120,7 @@ Environment="DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk www.sort-web-app.shef. ```bash sudo systemctl status gunicorn sudo systemctl status nginx +sudo systemctl status postgresql ``` # View logs From 813669b94cc7b445fa161015a12bf0156230a6d8 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 11:19:48 +0000 Subject: [PATCH 26/79] Configure database using env vars https://docs.djangoproject.com/en/5.1/ref/settings/#databases --- SORT/settings.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 5c35469..3592e9e 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -14,7 +14,16 @@ from dotenv import load_dotenv import os -load_dotenv() # Load environment variables from .env file + +def string_to_boolean(s: str) -> bool: + """ + Check if the string value is 1, yes, or true. + """ + return s.casefold()[0] in {"1", "y", "t"} + + +# Load environment variables from .env file +load_dotenv(os.getenv('DJANGO_ENV_PATH')) # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -26,7 +35,7 @@ SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = string_to_boolean(os.getenv("DJANGO_DEBUG", "False")) ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '').split() @@ -80,15 +89,19 @@ }, ] -WSGI_APPLICATION = "SORT.wsgi.application" +WSGI_APPLICATION = os.getenv("DJANGO_WSGI_APPLICATION", "SORT.wsgi.application") # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases - DATABASES = { + # Set the database settings using environment variables, or default to a local SQLite database file. "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / 'db.sqlite3' + "ENGINE": os.getenv("DJANGO_DATABASE_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.getenv("DJANGO_DATABASE_NAME", BASE_DIR / 'db.sqlite3'), + "USER": os.getenv("DJANGO_DATABASE_USER"), + "PASSWORD": os.getenv("DJANGO_DATABASE_PASSWORD"), + "HOST": os.getenv("DJANGO_DATABASE_HOST"), + "PORT": os.getenv("DJANGO_DATABASE_PORT"), } } From 68b147f8a35387b18c930c19affebf9099ddc28f Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 12:00:06 +0000 Subject: [PATCH 27/79] Forward HTTP host to proxy https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header --- config/nginx/gunicorn.conf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 089e3c8..1518fc9 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -13,6 +13,15 @@ server { server_name sort-web-app.shef.ac.uk www.sort-web-app.shef.ac.uk; keepalive_timeout 5; + # /server-status endpoint + # This is used by IT Services to monitor servers using collectd + # https://nginx.org/en/docs/http/ngx_http_stub_status_module.html + # https://www.collectd.org/documentation/manpages/collectd.conf.html + # It's based on Apache mod_status https://httpd.apache.org/docs/2.4/mod/mod_status.html + location = /server-status { + stub_status; + } + # Serve static files without invoking Python WSGI location /static/ { # https://nginx.org/en/docs/http/ngx_http_core_module.html#root @@ -21,9 +30,11 @@ server { # Proxy forward to the WSGI Python app location / { + # Set HTTP headers for the proxied service + # https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; + proxy_set_header Host $host; # we don't want nginx trying to do something clever with # redirects, we set the Host: header above already. proxy_redirect off; From e74e8d93db11a1b0a42197b76a6483643c931740 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 12:21:35 +0000 Subject: [PATCH 28/79] Update deployment docs --- README.md | 3 +- docs/deployment.md | 96 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 349dec7..956e721 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,9 @@ python manage.py createsuperuser 6. Create a `.env` file in the project root directory and add the following environment variables: -``` +```bash DJANGO_SECRET_KEY=your_secret_key +DJANGO_DEBUG=True ``` --- diff --git a/docs/deployment.md b/docs/deployment.md index e4a85a0..8b5f56a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,6 +1,6 @@ # Deployment -The production web server has the following architecture: +The production web server has the following architecture, which is a commonly-used and reasonably secure production setup for Django. ```mermaid --- @@ -21,14 +21,15 @@ end When accessing the web site, the following process happens: 1. A user uses their web browser to access the server using the HTTPS port 443; -2. The request is sent to the web server and is handled by Nginx; -3. Nginx uses the web server gateway interface (WSGI) to access the Django application; -4. Django uses the PostgreSQL database to store data. +2. The request is sent to the web server and is handled by Nginx, which acts as a ["reverse proxy" server](https://serverfault.com/a/331263); +3. Nginx uses the web server gateway interface (WSGI) to access Gunicorn, which serves the Django application; +4. Gunicorn spawns several workers to run the Python code that operates the website; +5. Django uses the PostgreSQL database to store data. You may also refer to the following guides: * Django documentation: [How to deploy Django](https://docs.djangoproject.com/en/5.1/howto/deployment/) -* [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) +* [Deploying Gunicorn](https://docs.gunicorn.org/en/latest/deploy.html) using nginx The relevant files are: @@ -42,6 +43,8 @@ This app can be deployed to a web server using the script [`deploy.sh`](../deplo 2. Run the deployment script: `sudo bash -x deploy.sh` 3. Configure the database +We can run commands and Bash scripts as the superuser (`root`) using the [`sudo` command](https://manpages.ubuntu.com/manpages/noble/en/man8/sudo.8.html). + # Configuration To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service. @@ -53,22 +56,41 @@ sudo mkdir --parents /opt/sort sudo nano /opt/sort/.env ``` -This file would typically look similar to this: +This file would typically look similar to the example below. The contents of these values should be stored in a password manager. ```ini DJANGO_SECRET_KEY=******** DJANGO_ALLOWED_HOSTS=sort-web-app.shef.ac.uk DJANGO_STATIC_ROOT=/var/www/sort/static +WSGI_APPLICATION=SORT.wsgi.application +DJANGO_SETTINGS_MODULE=SORT.settings +# Database settings +DJANGO_DATABASE_ENGINE=django.db.backends.postgresql +DJANGO_DATABASE_NAME=sort +DJANGO_DATABASE_USER=sort +DJANGO_DATABASE_PASSWORD=******** +DJANGO_DATABASE_HOST=127.0.0.1 +DJANGO_DATABASE_PORT=5432 +``` + +You can generate a secret key using the Python [secrets library](https://docs.python.org/3/library/secrets.html): + +```bash +python -c "import secrets; print(secrets.token_urlsafe(37))" ``` # Database installation +The database may be administered using command-line tools and SQL statements that are run as the `postgres` user. For more details, please refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/16/index.html) and [this guide](https://dev.to/matthewhegarty/postgresql-better-security-for-django-applications-3c7m). + To run these commands, switch to the `postgres` user: ```bash sudo su - postgres ``` +The [`su` command](https://manpages.ubuntu.com/manpages/noble/man1/su.1.html) creates a new shell on behalf of the `postgres` user. + ## Create a database [Create a database](https://www.postgresql.org/docs/16/tutorial-createdb.html) @@ -79,7 +101,7 @@ createdb sort ## Create a user -The app needs credentials to access the database. +The SORT app needs credentials to access the database. We'll create a database user that the application will use to read and write data. Create a user: @@ -87,6 +109,66 @@ Create a user: createuser sort ``` +Set the password for that user using the [psql tool](https://manpages.ubuntu.com/manpages/noble/man1/psql.1.html) which lets us run SQL queries on the PostgreSQL server. + +```bash +psql sort +``` + +```sql +ALTER USER sort WITH PASSWORD '********'; +``` + +We can list users (i.e. database "roles") using [the `\du` command](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMAND-DU) in PostgreSQL. + +## Grant permissions + +We must allow this user the minimum necessary privileges to operate the web app. We authorise the user using the PostgreSQL [grant statement](https://www.postgresql.org/docs/current/sql-grant.html), which we execute in the `psql` tool. + +Create a schema, which is a "folder" in the database (a namespace) that will contain our tables. + +```sql +CREATE SCHEMA sort AUTHORIZATION sort; +``` + +You can view the [list of schemas](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMAND-DN) using this PostgreSQL command: + +```sql +\dnS +``` + +The result looks something like this: + +``` +sort=# \dnS + List of schemas + Name | Owner +--------------------+------------------- + information_schema | postgres + pg_catalog | postgres + pg_toast | postgres + public | pg_database_owner + sort | sort +(5 rows) +``` + +Let's restrict the visibility of the schema so the app can only see the `sort` schema. + +```sql +ALTER ROLE sort SET SEARCH_PATH TO sort; +``` + +Let's allow the SORT app to read and write data to the database tables, including any new tables that are created. + +```sql +GRANT CONNECT ON DATABASE sort TO sort; +GRANT USAGE ON SCHEMA sort TO sort; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA sort TO sort; +ALTER DEFAULT PRIVILEGES FOR USER sort GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO sort; +``` + +On our PostgreSQL instance, this should create a database named `sort` with a user named `sort` that has all the necessary permissions on the `sort` schema to create, modify, and drop tables and read/write data to those tables. + # Management To use the Django management tool From 4a4b8b2ebab20bbc3c63a44062263ddf54834544 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 14:34:15 +0000 Subject: [PATCH 29/79] Create UTF-8 en_GB database localisation --- deploy.sh | 11 +++++++++-- docs/deployment.md | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/deploy.sh b/deploy.sh index 77faeea..af8eb3e 100644 --- a/deploy.sh +++ b/deploy.sh @@ -18,16 +18,21 @@ pip="$venv_dir/bin/pip" python_version="python3.12" python="$venv_dir/bin/python" +# Install locale +sudo locale-gen en_GB +sudo locale-gen en_GB.UTF-8 +sudo update-locale + # Create Python virtual environment apt update -qq -apt install --yes -qq "$python_version" "$python_version-venv" +apt install --upgrade --yes -qq "$python_version" "$python_version-venv" python3 -m venv "$venv_dir" # Install the SORT Django app package $pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" -# Install static files +# Install static files into DJANGO_STATIC_ROOT (cd "$sort_dir" && exec $python manage.py collectstatic --no-input) # Install Gunicorn service @@ -53,3 +58,5 @@ systemctl reload nginx.service # Install PostgreSQL database # https://ubuntu.com/server/docs/install-and-configure-postgresql apt install --yes -qq postgresql +# Restart PostgreSQL to enable any new locales +systemctl restart postgresql diff --git a/docs/deployment.md b/docs/deployment.md index 8b5f56a..3a76cf1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -93,12 +93,14 @@ The [`su` command](https://manpages.ubuntu.com/manpages/noble/man1/su.1.html) c ## Create a database -[Create a database](https://www.postgresql.org/docs/16/tutorial-createdb.html) +[Create a database](https://www.postgresql.org/docs/16/tutorial-createdb.html) with the appropriate [encoding](https://www.postgresql.org/docs/current/multibyte.html). ```bash -createdb sort +createdb --template=template0 --encoding=UTF8 --locale=en_GB.UTF-8 sort "SORT application" ``` +We can list databases using `psql --list`. + ## Create a user The SORT app needs credentials to access the database. We'll create a database user that the application will use to read and write data. @@ -171,13 +173,16 @@ On our PostgreSQL instance, this should create a database named `sort` with a us # Management -To use the Django management tool +To use the Django management tool, we need to load up the virtual environment of the SORT Django application and navigate to the directory containing the tool. ```bash sort_dir="/opt/sort" venv_dir="$sort_dir/venv" python="$venv_dir/bin/python" cd "$sort_dir" +# Check the Django management tool works +$python "$sort_dir"/manage.py version +# View available commands $python "$sort_dir"/manage.py help ``` @@ -193,7 +198,11 @@ Create a super-user sudo $python manage.py createsuperuser ``` +Load data +```bash +sudo $python manage.py loaddata data/*.json +``` # Monitoring From 19af2bedbbf7434c0a6ce06a61575e7536e52ab4 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 14:37:11 +0000 Subject: [PATCH 30/79] Ensure gunicorn is started --- deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index af8eb3e..224e6c0 100644 --- a/deploy.sh +++ b/deploy.sh @@ -18,7 +18,8 @@ pip="$venv_dir/bin/pip" python_version="python3.12" python="$venv_dir/bin/python" -# Install locale +# Install British UTF-8 locale so we can use this with PostgreSQL. +# This is important to avoid the limitations of the LATIN1 character set. sudo locale-gen en_GB sudo locale-gen en_GB.UTF-8 sudo update-locale @@ -41,6 +42,7 @@ cp --verbose config/systemd/gunicorn.socket /etc/systemd/system/gunicorn.socket systemctl daemon-reload systemctl enable gunicorn.service systemctl enable gunicorn.socket +systemctl start gunicorn.service systemctl reload gunicorn.service # Install web reverse proxy server From 7924f60c3d1ac8e512616a733e1f7a136a86288b Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Thu, 9 Jan 2025 14:42:27 +0000 Subject: [PATCH 31/79] - --- docs/deployment.md | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 3a76cf1..099c96c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -123,11 +123,17 @@ ALTER USER sort WITH PASSWORD '********'; We can list users (i.e. database "roles") using [the `\du` command](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMAND-DU) in PostgreSQL. +```bash +psql sort --command "\du" +``` + + + ## Grant permissions We must allow this user the minimum necessary privileges to operate the web app. We authorise the user using the PostgreSQL [grant statement](https://www.postgresql.org/docs/current/sql-grant.html), which we execute in the `psql` tool. -Create a schema, which is a "folder" in the database (a namespace) that will contain our tables. +Create a schema, which is a "folder" in the database (a namespace) that will contain our tables. Remember to initialise the PostgreSQL command line with `psql sort`. ```sql CREATE SCHEMA sort AUTHORIZATION sort; @@ -139,21 +145,6 @@ You can view the [list of schemas](https://www.postgresql.org/docs/current/app-p \dnS ``` -The result looks something like this: - -``` -sort=# \dnS - List of schemas - Name | Owner ---------------------+------------------- - information_schema | postgres - pg_catalog | postgres - pg_toast | postgres - public | pg_database_owner - sort | sort -(5 rows) -``` - Let's restrict the visibility of the schema so the app can only see the `sort` schema. ```sql @@ -173,7 +164,7 @@ On our PostgreSQL instance, this should create a database named `sort` with a us # Management -To use the Django management tool, we need to load up the virtual environment of the SORT Django application and navigate to the directory containing the tool. +To use the [Django management tool](https://docs.djangoproject.com/en/5.1/ref/django-admin/), we need to load up the virtual environment of the SORT Django application and navigate to the directory containing the tool. ```bash sort_dir="/opt/sort" @@ -182,7 +173,11 @@ python="$venv_dir/bin/python" cd "$sort_dir" # Check the Django management tool works $python "$sort_dir"/manage.py version -# View available commands +``` + +View available commands + +```bash $python "$sort_dir"/manage.py help ``` From e5e994839bf9b64cc1c648c1543eb235c1fa437a Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:33:33 +0000 Subject: [PATCH 32/79] String-to-Boolean: accept any input data type --- SORT/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 3592e9e..49265d6 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -12,14 +12,15 @@ from pathlib import Path from dotenv import load_dotenv +from typing import Any import os -def string_to_boolean(s: str) -> bool: +def string_to_boolean(obj: Any) -> bool: """ Check if the string value is 1, yes, or true. """ - return s.casefold()[0] in {"1", "y", "t"} + return str(obj).casefold()[0] in {"1", "y", "t"} # Load environment variables from .env file From 051bd7a6ef20ee7940924bb8a5da7130061ac357 Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 10 Jan 2025 10:51:01 +0000 Subject: [PATCH 33/79] fixes a couple of bugs --- home/forms.py | 3 +++ home/views.py | 8 +++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/home/forms.py b/home/forms.py index 205a491..619b96a 100644 --- a/home/forms.py +++ b/home/forms.py @@ -43,6 +43,9 @@ class Meta: def clean_email(self): email = self.cleaned_data.get('email') + if email == self.instance.email: + return email + if User.objects.exclude(pk=self.instance.pk).filter(email=email).exists(): raise forms.ValidationError("This email is already in use.") return email diff --git a/home/views.py b/home/views.py index 555a2d4..8186a5c 100644 --- a/home/views.py +++ b/home/views.py @@ -3,7 +3,6 @@ from django.contrib.auth.views import LoginView, LogoutView from django.views.generic.edit import CreateView, UpdateView from django.shortcuts import redirect -from survey.models import Survey from django.shortcuts import render from django.views import View from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm @@ -43,11 +42,10 @@ def dispatch(self, request, *args, **kwargs): class HomeView(LoginRequiredMixin, View): template_name = 'home/welcome.html' login_url = 'login' - def get(self, request): - consent_questionnaire = Survey.objects.get( - title="Consent") - return render(request, 'home/welcome.html', {'questionnaire': consent_questionnaire}) + def get(self, request, *args, **kwargs): + + return render(request, self.template_name) class ProfileView(LoginRequiredMixin, UpdateView): model = User form_class = UserProfileForm From c803dae835a05d20a189c49dff672d9579fe47ea Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Fri, 10 Jan 2025 10:57:47 +0000 Subject: [PATCH 34/79] Run linting on PRs and main branch only --- .github/workflows/lint-nginx.yaml | 6 +++++- .github/workflows/lint-shell-scripts.yaml | 6 +++++- .github/workflows/lint-systemd.yaml | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-nginx.yaml b/.github/workflows/lint-nginx.yaml index 4f4e974..781682f 100644 --- a/.github/workflows/lint-nginx.yaml +++ b/.github/workflows/lint-nginx.yaml @@ -1,7 +1,11 @@ # GitHub Actions workflow for validating NGINX configuration files # https://github.com/jhinch/nginx-linter name: Lint NGINX config files -on: [ push ] +on: + pull_request: + push: + branches: + - main jobs: lint: runs-on: ubuntu-24.04 diff --git a/.github/workflows/lint-shell-scripts.yaml b/.github/workflows/lint-shell-scripts.yaml index f09e8b8..94a52c8 100644 --- a/.github/workflows/lint-shell-scripts.yaml +++ b/.github/workflows/lint-shell-scripts.yaml @@ -2,7 +2,11 @@ # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions # https://github.com/marketplace/actions/shell-linter name: Lint shell scripts -on: push +on: + pull_request: + push: + branches: + - main jobs: lint_shell: runs-on: ubuntu-24.04 diff --git a/.github/workflows/lint-systemd.yaml b/.github/workflows/lint-systemd.yaml index 73db1d0..1dec7b6 100644 --- a/.github/workflows/lint-systemd.yaml +++ b/.github/workflows/lint-systemd.yaml @@ -1,7 +1,11 @@ # GitHub Actions workflow for linting the systemd unit files # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions name: Lint systemd units -on: [ push ] +on: + pull_request: + push: + branches: + - main jobs: lint: runs-on: ubuntu-24.04 From e70549dd23ee0bdb52db088d83f1700e8ae61a29 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Fri, 10 Jan 2025 11:00:49 +0000 Subject: [PATCH 35/79] Lock down the personality(2) system call /home/runner/work/SORT/SORT/config/systemd/gunicorn.service:1:error [Security.LockPersonality] - Service should have LockPersonality being set https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html --- config/systemd/gunicorn.service | 1 + 1 file changed, 1 insertion(+) diff --git a/config/systemd/gunicorn.service b/config/systemd/gunicorn.service index 4483ef9..7845ea6 100644 --- a/config/systemd/gunicorn.service +++ b/config/systemd/gunicorn.service @@ -26,6 +26,7 @@ PrivateTmp=true # if your app does not need administrative capabilities, let systemd know ProtectSystem=strict Delegate=yes +LockPersonality=yes [Install] WantedBy=multi-user.target From 2976280663ed5594dec21046772c850d28a5d441 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 09:54:37 +0000 Subject: [PATCH 36/79] Add deployment check tool https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ --- deploy.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy.sh b/deploy.sh index 224e6c0..f22cb1d 100644 --- a/deploy.sh +++ b/deploy.sh @@ -34,6 +34,7 @@ $pip install --quiet -r requirements.txt cp --recursive * "$sort_dir/" # Install static files into DJANGO_STATIC_ROOT +# This runs in a subshell because it's changing directory (cd "$sort_dir" && exec $python manage.py collectstatic --no-input) # Install Gunicorn service @@ -62,3 +63,7 @@ systemctl reload nginx.service apt install --yes -qq postgresql # Restart PostgreSQL to enable any new locales systemctl restart postgresql + +# Run deployment checks +# https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ +(cd "$sort_dir" && exec $python manage.py check --deploy) From 8a003925829765d287de2b9e8c1174c3c004a115 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 11:22:47 +0000 Subject: [PATCH 37/79] Add CONTRIBUTING.md --- CONTRIBUTING.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..fe41b7a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,11 @@ +# Contribution guide + +We welcome contributions to SORT! This document outlines the guidelines for contributing to the project. + +# Getting Started + +... + +# Code of Conduct + +We expect all contributors to follow the SORT [Code of Conduct](CODE_OF_CONDUCT.md). From 7099b1fb21195ffcdf69dd33dacff0d97cf7e089 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:15:41 +0000 Subject: [PATCH 38/79] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe41b7a..d880a0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,8 @@ We welcome contributions to SORT! This document outlines the guidelines for cont ... +https://www.conventionalcommits.org/en/v1.0.0/#summary + # Code of Conduct We expect all contributors to follow the SORT [Code of Conduct](CODE_OF_CONDUCT.md). From 59467c410c9d237c94ff0d16b9f54932c0dd8080 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 14:32:06 +0000 Subject: [PATCH 39/79] Add cookie security settings --- SORT/settings.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 49265d6..0833ec0 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -16,7 +16,7 @@ import os -def string_to_boolean(obj: Any) -> bool: +def cast_to_boolean(obj: Any) -> bool: """ Check if the string value is 1, yes, or true. """ @@ -36,9 +36,9 @@ def string_to_boolean(obj: Any) -> bool: SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = string_to_boolean(os.getenv("DJANGO_DEBUG", "False")) +DEBUG = cast_to_boolean(os.getenv("DJANGO_DEBUG", "False")) -ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', '').split() +ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', 'sort-web-app.shef.ac.uk').split() # Application definition @@ -61,7 +61,9 @@ def string_to_boolean(obj: Any) -> bool: ] MIDDLEWARE = [ - "django.middleware.security.SecurityMiddleware", + # Implement security in the web server, not in Django. + # https://docs.djangoproject.com/en/5.1/ref/middleware/#module-django.middleware.security + # "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -178,3 +180,7 @@ def string_to_boolean(obj: Any) -> bool: # EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" STATIC_ROOT = os.getenv('DJANGO_STATIC_ROOT') + +# Security settings +SESSION_COOKIE_SECURE = cast_to_boolean(os.getenv("DJANGO_SESSION_COOKIE_SECURE", not DEBUG)) +CSRF_COOKIE_SECURE = cast_to_boolean(os.getenv("DJANGO_CSRF_COOKIE_SECURE", not DEBUG)) From fbe965a6dabd251f391f01e6d7dff4250cd876e9 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 14:40:42 +0000 Subject: [PATCH 40/79] Make to-Boolean function more robust --- SORT/settings.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/SORT/settings.py b/SORT/settings.py index 0833ec0..43436db 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -19,8 +19,16 @@ def cast_to_boolean(obj: Any) -> bool: """ Check if the string value is 1, yes, or true. + + Empty values are interpreted as False. """ - return str(obj).casefold()[0] in {"1", "y", "t"} + # Cast to lower case string + obj = str(obj).casefold() + # False / off + if obj in {"", "off", "none"}: + return False + # True / on + return obj[0] in {"1", "y", "t", "o"} # Load environment variables from .env file From 13b96cfa5d9c1c7ee96fc990ed07e8ce9459388b Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 14:40:52 +0000 Subject: [PATCH 41/79] Make to-Boolean function more robust --- docs/deployment.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 099c96c..2f72f33 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -47,7 +47,9 @@ We can run commands and Bash scripts as the superuser (`root`) using the [`sudo` # Configuration -To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service. +## Environment variables + +To configure the environment variables for the service, you can either edit the `.env` file and/or add them to the systemd service using `systemctl edit`. To edit the environment file: @@ -56,7 +58,7 @@ sudo mkdir --parents /opt/sort sudo nano /opt/sort/.env ``` -This file would typically look similar to the example below. The contents of these values should be stored in a password manager. +This file would typically look similar to the example below. The contents of these values should be stored in a password manager. In general, the name of each environment variable is the same as the [Django setting](https://docs.djangoproject.com/en/5.1/ref/settings) with the prefix `DJANGO_`. ```ini DJANGO_SECRET_KEY=******** @@ -76,9 +78,27 @@ DJANGO_DATABASE_PORT=5432 You can generate a secret key using the Python [secrets library](https://docs.python.org/3/library/secrets.html): ```bash -python -c "import secrets; print(secrets.token_urlsafe(37))" +python -c "import secrets; print(secrets.token_urlsafe())" +``` + +## Service settings + +If needed, you can add environment variables to the `systemd` service like so: + +```bash +sudo systemctl edit gunicorn.service ``` +And add environment variables, or other `systemd` settings, to that override configuration file: + +``` +[Service] +Environment="DJANGO_SECRET_KEY=********" +Environment="DEBUG=off" +``` + + + # Database installation The database may be administered using command-line tools and SQL statements that are run as the `postgres` user. For more details, please refer to the [PostgreSQL documentation](https://www.postgresql.org/docs/16/index.html) and [this guide](https://dev.to/matthewhegarty/postgresql-better-security-for-django-applications-3c7m). @@ -89,7 +109,7 @@ To run these commands, switch to the `postgres` user: sudo su - postgres ``` -The [`su` command](https://manpages.ubuntu.com/manpages/noble/man1/su.1.html) creates a new shell on behalf of the `postgres` user. +The [`su` command](https://manpages.ubuntu.com/manpages/noble/man1/su.1.html) creates a new shell on behalf of the `postgres` user. ## Create a database From 0e7a2a2cdb93d78e7eff93deada73723bb48fb90 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 14:45:48 +0000 Subject: [PATCH 42/79] SECRET_KEY has min length 50 chars --- docs/deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 2f72f33..08c6313 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -78,7 +78,7 @@ DJANGO_DATABASE_PORT=5432 You can generate a secret key using the Python [secrets library](https://docs.python.org/3/library/secrets.html): ```bash -python -c "import secrets; print(secrets.token_urlsafe())" +python -c "import secrets; print(secrets.token_urlsafe(37))" ``` ## Service settings From 16af9558801d4a6e0e0c32ba2c644250026932f4 Mon Sep 17 00:00:00 2001 From: f-allian Date: Mon, 13 Jan 2025 15:51:16 +0000 Subject: [PATCH 43/79] Exclude migrations in workflow --- .github/workflows/lint-python.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-python.yaml b/.github/workflows/lint-python.yaml index b7f3c88..3ab70fb 100644 --- a/.github/workflows/lint-python.yaml +++ b/.github/workflows/lint-python.yaml @@ -14,9 +14,9 @@ jobs: - name: Lint Python code run: | dirs="SORT home survey invites" - isort $dirs - black $dirs - flake8 $dirs + isort $dirs --skip-glob '*/migrations/*' + black $dirs --exclude '/migrations/' + flake8 $dirs --exclude '*/migrations/*' # Suggest merging any changes - name: Create Pull Request # https://github.com/marketplace/actions/create-pull-request From 5b19e897b6d2db3203623147fb06eb9edf0f7623 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:03:09 +0000 Subject: [PATCH 44/79] Add SSL cert config --- config/nginx/gunicorn.conf | 24 ++++++++++++++++++++---- docs/deployment.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 1518fc9..63df91f 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -1,4 +1,12 @@ +# nginx configuration file + +# HTTP and SSL certificates +# https://nginx.org/en/docs/http/configuring_https_servers.html +# https://ssl-config.mozilla.org/#server=nginx&version=1.27.3&config=modern&openssl=3.4.0&ocsp=false&guideline=5.7 + +# Gunicorn proxy configuration # https://docs.gunicorn.org/en/stable/deploy.html#nginx-configuration + upstream app_server { # fail_timeout=0 means we always retry an upstream even if it failed # to return a good HTTP response @@ -7,10 +15,18 @@ upstream app_server { } server { - # https://nginx.org/en/docs/http/ngx_http_core_module.html#listen - listen 80 deferred; - client_max_body_size 4G; - server_name sort-web-app.shef.ac.uk www.sort-web-app.shef.ac.uk; + # https://nginx.org/en/docs/http/ngx_http_core_module.html + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + # SSL options + ssl_certificate /etc/ssl/certs/sort.crt; + ssl_certificate_key /etc/ssl/private/sort.key; + ssl_protocols TLSv1.3; + ssl_ecdh_curve X25519:prime256v1:secp384r1; + ssl_prefer_server_ciphers off; + client_max_body_size 1m; + server_name sort-web-app.shef.ac.uk; keepalive_timeout 5; # /server-status endpoint diff --git a/docs/deployment.md b/docs/deployment.md index 08c6313..56a9542 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -182,6 +182,34 @@ ALTER DEFAULT PRIVILEGES FOR USER sort GRANT SELECT, INSERT, UPDATE, DELETE ON T On our PostgreSQL instance, this should create a database named `sort` with a user named `sort` that has all the necessary permissions on the `sort` schema to create, modify, and drop tables and read/write data to those tables. +# Security + +## SSL Certificates + +See: ITS Wiki [SSL Certificates/Howto](https://itswiki.shef.ac.uk/wiki/SSL_Certificates/Howto) for the commands to generate a Certificate Signing Request (CSR) using [OpenSSL](https://docs.openssl.org/3.3/man1/openssl-req/#options) with an unencrypted private key. + +We can install the private key + +```bash +sudo mv "$(hostname -s)_shef_ac_uk.key" /etc/ssl/private/sort.key +sudo chmod 640 /etc/ssl/private/sort.key +sudo chown root:ssl-cert /etc/ssl/private/sort.key +``` + +The CSR may be used to get a signed SSL certificate via [Information Security](https://staff.sheffield.ac.uk/it-services/information-security) in IT Services. + +For *development purposes only* we can generate a self-signed certificate + +```bash +openssl x509 -signkey /etc/ssl/private/sort.key -in "$(hostname -s)_shef_ac_uk.csr" -req -days 365 -out "$(hostname -s)_shef_ac_uk.crt" +``` + +Install the SSL certificate: + +```bash +sudo cp "$(hostname -s)_shef_ac_uk.crt" /etc/ssl/certs/sort.pem +``` + # Management To use the [Django management tool](https://docs.djangoproject.com/en/5.1/ref/django-admin/), we need to load up the virtual environment of the SORT Django application and navigate to the directory containing the tool. From e29cbf4ab519e926f166440ae654588bbf26addd Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:04:38 +0000 Subject: [PATCH 45/79] Suggest Python version 3.12 because it's used on Ubuntu 24 LTS --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 956e721..5354ab5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Follow these steps to set up and run the app locally: Prerequisites -- Python 3.10 +- Python 3.12 - pip --- From 4d3adc240ed4ed9be824a0022d90cabc11873de7 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:06:59 +0000 Subject: [PATCH 46/79] Disable HTTP2 https://nginx.org/en/docs/http/ngx_http_v2_module.html --- config/nginx/gunicorn.conf | 4 +++- deploy.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/nginx/gunicorn.conf b/config/nginx/gunicorn.conf index 63df91f..e95cd5c 100644 --- a/config/nginx/gunicorn.conf +++ b/config/nginx/gunicorn.conf @@ -18,7 +18,9 @@ server { # https://nginx.org/en/docs/http/ngx_http_core_module.html listen 443 ssl; listen [::]:443 ssl; - http2 on; + # Unavailable on nginx versions before 1.25.1 + # https://nginx.org/en/docs/http/ngx_http_v2_module.html + #http2 on; # SSL options ssl_certificate /etc/ssl/certs/sort.crt; ssl_certificate_key /etc/ssl/private/sort.key; diff --git a/deploy.sh b/deploy.sh index f22cb1d..f38c5f6 100644 --- a/deploy.sh +++ b/deploy.sh @@ -50,6 +50,7 @@ systemctl reload gunicorn.service # Install nginx # https://nginx.org/en/docs/install.html apt install --yes -qq nginx +nginx -version # Configure web server rm -f /etc/nginx/sites-enabled/default From b4979276b5e2c94d296f2b33c8421c7dcd4db518 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:11:22 +0000 Subject: [PATCH 47/79] Add note on self-signed certificates --- docs/deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 56a9542..45b0bf6 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -198,7 +198,7 @@ sudo chown root:ssl-cert /etc/ssl/private/sort.key The CSR may be used to get a signed SSL certificate via [Information Security](https://staff.sheffield.ac.uk/it-services/information-security) in IT Services. -For *development purposes only* we can generate a self-signed certificate +For *development purposes only* we can generate a self-signed certificate (which will cause web browsers to say "Not secure") ```bash openssl x509 -signkey /etc/ssl/private/sort.key -in "$(hostname -s)_shef_ac_uk.csr" -req -days 365 -out "$(hostname -s)_shef_ac_uk.crt" From bc19e29876824035dd5337dd105a07527f547665 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:12:56 +0000 Subject: [PATCH 48/79] Format file --- SORT/settings.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 43436db..0b7927c 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -10,10 +10,11 @@ https://docs.djangoproject.com/en/5.1/ref/settings/ """ +import os from pathlib import Path -from dotenv import load_dotenv from typing import Any -import os + +from dotenv import load_dotenv def cast_to_boolean(obj: Any) -> bool: @@ -32,7 +33,7 @@ def cast_to_boolean(obj: Any) -> bool: # Load environment variables from .env file -load_dotenv(os.getenv('DJANGO_ENV_PATH')) +load_dotenv(os.getenv("DJANGO_ENV_PATH")) # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -41,12 +42,12 @@ def cast_to_boolean(obj: Any) -> bool: # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') +SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = cast_to_boolean(os.getenv("DJANGO_DEBUG", "False")) -ALLOWED_HOSTS = os.getenv('DJANGO_ALLOWED_HOSTS', 'sort-web-app.shef.ac.uk').split() +ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "sort-web-app.shef.ac.uk").split() # Application definition @@ -59,10 +60,8 @@ def cast_to_boolean(obj: Any) -> bool: "django.contrib.staticfiles", "django_bootstrap5", "django_extensions", - 'debug_toolbar', - + "debug_toolbar", # apps created by FA: - "home", "survey", "invites", @@ -78,7 +77,7 @@ def cast_to_boolean(obj: Any) -> bool: "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - 'debug_toolbar.middleware.DebugToolbarMiddleware' + "debug_toolbar.middleware.DebugToolbarMiddleware", ] ROOT_URLCONF = "SORT.urls" @@ -86,8 +85,7 @@ def cast_to_boolean(obj: Any) -> bool: TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [BASE_DIR / 'static/templates'] - , + "DIRS": [BASE_DIR / "static/templates"], "APP_DIRS": True, "OPTIONS": { "context_processors": [ @@ -108,7 +106,7 @@ def cast_to_boolean(obj: Any) -> bool: # Set the database settings using environment variables, or default to a local SQLite database file. "default": { "ENGINE": os.getenv("DJANGO_DATABASE_ENGINE", "django.db.backends.sqlite3"), - "NAME": os.getenv("DJANGO_DATABASE_NAME", BASE_DIR / 'db.sqlite3'), + "NAME": os.getenv("DJANGO_DATABASE_NAME", BASE_DIR / "db.sqlite3"), "USER": os.getenv("DJANGO_DATABASE_USER"), "PASSWORD": os.getenv("DJANGO_DATABASE_PASSWORD"), "HOST": os.getenv("DJANGO_DATABASE_HOST"), @@ -150,7 +148,7 @@ def cast_to_boolean(obj: Any) -> bool: STATIC_URL = "static/" -STATICFILES_DIRS = [BASE_DIR / 'static'] +STATICFILES_DIRS = [BASE_DIR / "static"] # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -170,11 +168,9 @@ def cast_to_boolean(obj: Any) -> bool: # FA: for local testing emails: -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" -AUTHENTICATION_BACKENDS = ( - 'django.contrib.auth.backends.ModelBackend', -) +AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",) # For django-debug-toolbar INTERNAL_IPS = [ @@ -187,8 +183,10 @@ def cast_to_boolean(obj: Any) -> bool: # EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" -STATIC_ROOT = os.getenv('DJANGO_STATIC_ROOT') +STATIC_ROOT = os.getenv("DJANGO_STATIC_ROOT") # Security settings -SESSION_COOKIE_SECURE = cast_to_boolean(os.getenv("DJANGO_SESSION_COOKIE_SECURE", not DEBUG)) +SESSION_COOKIE_SECURE = cast_to_boolean( + os.getenv("DJANGO_SESSION_COOKIE_SECURE", not DEBUG) +) CSRF_COOKIE_SECURE = cast_to_boolean(os.getenv("DJANGO_CSRF_COOKIE_SECURE", not DEBUG)) From 018f2b5a9f5c0e6a4d8cbcd27c026a94379b2c00 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:27:12 +0000 Subject: [PATCH 49/79] Put app URLconfs in root urlconf https://docs.djangoproject.com/en/5.1/topics/http/urls/#including-other-urlconfs --- SORT/urls.py | 2 ++ home/urls.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SORT/urls.py b/SORT/urls.py index 2e450d4..5825e62 100644 --- a/SORT/urls.py +++ b/SORT/urls.py @@ -22,6 +22,8 @@ urlpatterns = [ path("admin/", admin.site.urls), path("", include("home.urls")), + path("invite/", include("invites.urls"), name="invites"), + path("", include("survey.urls"), name="survey"), ] if settings.DEBUG: diff --git a/home/urls.py b/home/urls.py index 93357c9..29e2706 100644 --- a/home/urls.py +++ b/home/urls.py @@ -9,8 +9,6 @@ path("login/", views.LoginInterfaceView.as_view(), name="login"), path("logout/", views.LogoutInterfaceView.as_view(), name="logout"), path("signup/", views.SignupView.as_view(), name="signup"), - path("", include("survey.urls"), name="survey"), - path("invite/", include("invites.urls"), name="invites"), path("profile/", views.ProfileView.as_view(), name="profile"), path( "password_reset/", From c0b8b529a9cc09be56d52f6de654d63f8d84755e Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:34:44 +0000 Subject: [PATCH 50/79] Set internationalisation options https://docs.djangoproject.com/en/5.1/topics/i18n/ --- SORT/settings.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 2eef0c0..6bc1024 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -20,7 +20,6 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ @@ -32,7 +31,6 @@ ALLOWED_HOSTS = [] - # Application definition INSTALLED_APPS = [ @@ -82,7 +80,6 @@ WSGI_APPLICATION = "SORT.wsgi.application" - # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases @@ -116,19 +113,18 @@ }, ] - # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ -LANGUAGE_CODE = "en-us" +LANGUAGE_CODE = "en-gb" TIME_ZONE = "UTC" -USE_I18N = True +# https://docs.djangoproject.com/en/5.1/topics/i18n/translation/ +USE_I18N = False USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ @@ -136,7 +132,6 @@ STATICFILES_DIRS = [BASE_DIR / "static"] - # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -145,7 +140,6 @@ LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/" - # FA: End session when the browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = True @@ -166,7 +160,7 @@ "127.0.0.1", # ... ] -AUTH_USER_MODEL = 'home.User' # FA: replace username with email as unique identifiers +AUTH_USER_MODEL = 'home.User' # FA: replace username with email as unique identifiers # FA: for production: From 09a1733c2d6a92b4b10a504db822d6a4c13ef7f2 Mon Sep 17 00:00:00 2001 From: Joe Heffer Date: Mon, 13 Jan 2025 16:38:32 +0000 Subject: [PATCH 51/79] Set internationalisation options https://docs.djangoproject.com/en/5.1/topics/i18n/ --- SORT/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 2eef0c0..5a44649 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -120,11 +120,12 @@ # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ -LANGUAGE_CODE = "en-us" +LANGUAGE_CODE = "en-gb" -TIME_ZONE = "UTC" +TIME_ZONE = "Europe/London" -USE_I18N = True +# Disable translation features +USE_I18N = False USE_TZ = True From 3d2e0103270d5d826fbcbac84ae14c9f9a4d41b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:04:15 +0000 Subject: [PATCH 52/79] build(deps): bump django from 5.1.2 to 5.1.4 Bumps [django](https://github.com/django/django) from 5.1.2 to 5.1.4. - [Commits](https://github.com/django/django/compare/5.1.2...5.1.4) --- updated-dependencies: - dependency-name: django dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47d408e..5417512 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ asgiref==3.8.1 asttokens==2.4.1 colorama==0.4.6 decorator==5.1.1 -Django==5.1.2 +Django==5.1.4 django-bootstrap5==24.3 django-extensions==3.2.3 exceptiongroup==1.2.2 From 1c6d3dbd77c2dac7e2044ba4a6a4886a56e0a255 Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Tue, 14 Jan 2025 11:29:23 +0000 Subject: [PATCH 53/79] Fix error caused from merging dynamic forms code --- home/migrations/0001_initial.py | 100 ++++++++++++----- home/migrations/0002_remove_user_is_admin.py | 16 --- ...ame_is_activate_user_is_active_and_more.py | 64 ----------- ...004_project_alter_organisation_projects.py | 51 --------- ...name_firstname_user_first_name_and_more.py | 23 ---- ...6_remove_organisation_projects_and_more.py | 101 ------------------ home/models.py | 3 +- home/urls.py | 1 - home/views.py | 8 +- invites/admin.py | 3 - invites/forms.py | 11 -- survey/migrations/0001_initial.py | 4 +- survey/models.py | 5 +- 13 files changed, 82 insertions(+), 308 deletions(-) delete mode 100644 home/migrations/0002_remove_user_is_admin.py delete mode 100644 home/migrations/0003_rename_is_activate_user_is_active_and_more.py delete mode 100644 home/migrations/0004_project_alter_organisation_projects.py delete mode 100644 home/migrations/0005_rename_firstname_user_first_name_and_more.py delete mode 100644 home/migrations/0006_remove_organisation_projects_and_more.py delete mode 100644 invites/admin.py delete mode 100644 invites/forms.py diff --git a/home/migrations/0001_initial.py b/home/migrations/0001_initial.py index f025db7..b647204 100644 --- a/home/migrations/0001_initial.py +++ b/home/migrations/0001_initial.py @@ -1,42 +1,90 @@ -# Generated by Django 5.1.2 on 2024-11-28 15:17 +# Generated by Django 5.1.2 on 2025-01-14 11:28 +import django.db.models.deletion +from django.conf import settings from django.db import migrations, models class Migration(migrations.Migration): + initial = True - dependencies = [] + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] operations = [ migrations.CreateModel( - name="User", + name='Organisation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=200)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='User', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('first_name', models.CharField(max_length=50)), + ('last_name', models.CharField(max_length=50)), + ('email', models.EmailField(max_length=254, unique=True)), + ('is_active', models.BooleanField(default=True)), + ('is_superuser', models.BooleanField(default=False)), + ('is_staff', models.BooleanField(default=False)), + ('date_joined', models.DateTimeField(auto_now_add=True)), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='OrganisationMembership', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "last_login", - models.DateTimeField( - blank=True, null=True, verbose_name="last login" - ), - ), - ("firstname", models.CharField(max_length=50)), - ("lastname", models.CharField(max_length=50)), - ("email", models.EmailField(max_length=254, unique=True)), - ("password", models.CharField(max_length=100)), - ("organisation", models.CharField(max_length=200)), - ("is_activate", models.BooleanField(default=True)), - ("is_admin", models.BooleanField(default=False)), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('role', models.CharField(choices=[('ADMIN', 'Administrator'), ('MEMBER', 'Member'), ('GUEST', 'Guest')], default='GUEST', max_length=20)), + ('joined_at', models.DateTimeField(auto_now_add=True)), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.organisation')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], options={ - "abstract": False, + 'unique_together': {('user', 'organisation')}, }, ), + migrations.AddField( + model_name='organisation', + name='members', + field=models.ManyToManyField(through='home.OrganisationMembership', to=settings.AUTH_USER_MODEL), + ), + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('created_on', models.DateTimeField(auto_now_add=True)), + ('created_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='ProjectOrganisation', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('added_at', models.DateTimeField(auto_now_add=True)), + ('added_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.organisation')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.project')), + ], + options={ + 'unique_together': {('project', 'organisation')}, + }, + ), + migrations.AddField( + model_name='project', + name='organisations', + field=models.ManyToManyField(through='home.ProjectOrganisation', to='home.organisation'), + ), ] diff --git a/home/migrations/0002_remove_user_is_admin.py b/home/migrations/0002_remove_user_is_admin.py deleted file mode 100644 index 53e9324..0000000 --- a/home/migrations/0002_remove_user_is_admin.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-28 15:58 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("home", "0001_initial"), - ] - - operations = [ - migrations.RemoveField( - model_name="user", - name="is_admin", - ), - ] diff --git a/home/migrations/0003_rename_is_activate_user_is_active_and_more.py b/home/migrations/0003_rename_is_activate_user_is_active_and_more.py deleted file mode 100644 index 3d84506..0000000 --- a/home/migrations/0003_rename_is_activate_user_is_active_and_more.py +++ /dev/null @@ -1,64 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-28 16:52 - -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("home", "0002_remove_user_is_admin"), - ] - - operations = [ - migrations.RenameField( - model_name="user", - old_name="is_activate", - new_name="is_active", - ), - migrations.RemoveField( - model_name="user", - name="organisation", - ), - migrations.AddField( - model_name="user", - name="is_staff", - field=models.BooleanField(default=False), - ), - migrations.AddField( - model_name="user", - name="is_superuser", - field=models.BooleanField(default=False), - ), - migrations.CreateModel( - name="Organisation", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=200)), - ("projects", models.TextField(blank=True)), - ( - "users", - models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL), - ), - ], - ), - migrations.AddField( - model_name="user", - name="organisations", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="user", - to="home.organisation", - ), - ), - ] diff --git a/home/migrations/0004_project_alter_organisation_projects.py b/home/migrations/0004_project_alter_organisation_projects.py deleted file mode 100644 index 3ed9ba8..0000000 --- a/home/migrations/0004_project_alter_organisation_projects.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-28 17:04 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("home", "0003_rename_is_activate_user_is_active_and_more"), - ("survey", "0009_alter_answer_token_alter_comment_token"), - ] - - operations = [ - migrations.CreateModel( - name="Project", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=100)), - ("created_on", models.DateTimeField(auto_now_add=True)), - ( - "organisations", - models.ForeignKey( - blank=True, - on_delete=django.db.models.deletion.CASCADE, - to="home.organisation", - ), - ), - ( - "surveys", - models.ManyToManyField(blank=True, to="survey.questionnaire"), - ), - ], - ), - migrations.AlterField( - model_name="organisation", - name="projects", - field=models.ForeignKey( - blank=True, - on_delete=django.db.models.deletion.CASCADE, - to="home.project", - ), - ), - ] diff --git a/home/migrations/0005_rename_firstname_user_first_name_and_more.py b/home/migrations/0005_rename_firstname_user_first_name_and_more.py deleted file mode 100644 index 6cfe911..0000000 --- a/home/migrations/0005_rename_firstname_user_first_name_and_more.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.1.2 on 2025-01-10 12:33 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('home', '0004_project_alter_organisation_projects'), - ] - - operations = [ - migrations.RenameField( - model_name='user', - old_name='firstname', - new_name='first_name', - ), - migrations.RenameField( - model_name='user', - old_name='lastname', - new_name='last_name', - ), - ] diff --git a/home/migrations/0006_remove_organisation_projects_and_more.py b/home/migrations/0006_remove_organisation_projects_and_more.py deleted file mode 100644 index a8bcae1..0000000 --- a/home/migrations/0006_remove_organisation_projects_and_more.py +++ /dev/null @@ -1,101 +0,0 @@ -# Generated by Django 5.1.2 on 2025-01-10 13:12 - -import django.db.models.deletion -import django.utils.timezone -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('auth', '0012_alter_user_first_name_max_length'), - ('home', '0005_rename_firstname_user_first_name_and_more'), - ] - - operations = [ - migrations.RemoveField( - model_name='organisation', - name='projects', - ), - migrations.RemoveField( - model_name='organisation', - name='users', - ), - migrations.RemoveField( - model_name='user', - name='organisations', - ), - migrations.AddField( - model_name='organisation', - name='created_at', - field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), - preserve_default=False, - ), - migrations.AddField( - model_name='project', - name='created_by', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), - ), - migrations.AddField( - model_name='user', - name='date_joined', - field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), - preserve_default=False, - ), - migrations.AddField( - model_name='user', - name='groups', - field=models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups'), - ), - migrations.AddField( - model_name='user', - name='user_permissions', - field=models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions'), - ), - migrations.RemoveField( - model_name='project', - name='organisations', - ), - migrations.AlterField( - model_name='user', - name='password', - field=models.CharField(max_length=128, verbose_name='password'), - ), - migrations.CreateModel( - name='OrganisationMembership', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('role', models.CharField(choices=[('ADMIN', 'Administrator'), ('MEMBER', 'Member'), ('GUEST', 'Guest')], default='GUEST', max_length=20)), - ('joined_at', models.DateTimeField(auto_now_add=True)), - ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.organisation')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'unique_together': {('user', 'organisation')}, - }, - ), - migrations.AddField( - model_name='organisation', - name='members', - field=models.ManyToManyField(through='home.OrganisationMembership', to=settings.AUTH_USER_MODEL), - ), - migrations.CreateModel( - name='ProjectOrganisation', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('added_at', models.DateTimeField(auto_now_add=True)), - ('added_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), - ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.organisation')), - ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.project')), - ], - options={ - 'unique_together': {('project', 'organisation')}, - }, - ), - migrations.AddField( - model_name='project', - name='organisations', - field=models.ManyToManyField(through='home.ProjectOrganisation', to='home.organisation'), - ), - ] diff --git a/home/models.py b/home/models.py index 574f561..afc3edc 100644 --- a/home/models.py +++ b/home/models.py @@ -4,7 +4,7 @@ BaseUserManager, PermissionsMixin, ) -from survey.models import Questionnaire + class UserManager(BaseUserManager): @@ -91,7 +91,6 @@ class Project(models.Model): """ A project can be associated with multiple organisations/teams """ created_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True) created_on = models.DateTimeField(auto_now_add=True) - surveys = models.ManyToManyField(Questionnaire, blank=True) def __str__(self): return self.name diff --git a/home/urls.py b/home/urls.py index 93357c9..4a6ce55 100644 --- a/home/urls.py +++ b/home/urls.py @@ -10,7 +10,6 @@ path("logout/", views.LogoutInterfaceView.as_view(), name="logout"), path("signup/", views.SignupView.as_view(), name="signup"), path("", include("survey.urls"), name="survey"), - path("invite/", include("invites.urls"), name="invites"), path("profile/", views.ProfileView.as_view(), name="profile"), path( "password_reset/", diff --git a/home/views.py b/home/views.py index dcde05f..208fdd6 100644 --- a/home/views.py +++ b/home/views.py @@ -6,7 +6,6 @@ from django.views.generic.edit import CreateView, UpdateView from .models import Project, OrganisationMembership -from survey.models import Questionnaire from django.shortcuts import render from django.views import View from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm @@ -60,14 +59,9 @@ class HomeView(LoginRequiredMixin, View): login_url = "login" def get(self, request): - try: - consent_questionnaire = Questionnaire.objects.get(title="Consent") - print(consent_questionnaire) - except ObjectDoesNotExist: - consent_questionnaire = None return render( - request, self.template_name, {"questionnaire": consent_questionnaire} + request, self.template_name, {} ) diff --git a/invites/admin.py b/invites/admin.py deleted file mode 100644 index 4185d36..0000000 --- a/invites/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -# from django.contrib import admin - -# Register your models here. diff --git a/invites/forms.py b/invites/forms.py deleted file mode 100644 index b827777..0000000 --- a/invites/forms.py +++ /dev/null @@ -1,11 +0,0 @@ -from django import forms -from django.core.validators import EmailValidator - - -class InvitationForm(forms.Form): - email = forms.EmailField( - label="Participant Email", - max_length=100, - required=True, - validators=[EmailValidator()], - ) diff --git a/survey/migrations/0001_initial.py b/survey/migrations/0001_initial.py index 796716e..1b5a12b 100644 --- a/survey/migrations/0001_initial.py +++ b/survey/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-12-03 06:54 +# Generated by Django 5.1.2 on 2025-01-14 11:28 import django.db.models.deletion from django.db import migrations, models @@ -9,6 +9,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + ('home', '0001_initial'), ] operations = [ @@ -19,6 +20,7 @@ class Migration(migrations.Migration): ('title', models.CharField(max_length=200)), ('description', models.TextField()), ('survey_config', models.JSONField()), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.project')), ], ), migrations.CreateModel( diff --git a/survey/models.py b/survey/models.py index 2590897..489ed4a 100644 --- a/survey/models.py +++ b/survey/models.py @@ -2,6 +2,7 @@ from django.db import models from django.urls import reverse from django.utils import timezone +from home.models import Project class Survey(models.Model): """ @@ -10,7 +11,7 @@ class Survey(models.Model): title = models.CharField(max_length=200) description = models.TextField() survey_config = models.JSONField() - # TODO: Add the project it belongs to as foreign key + project = models.ForeignKey(Project, on_delete=models.CASCADE) def __str__(self): return self.title @@ -48,4 +49,4 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) def is_expired(self): - return timezone.now() > self.created_at + timezone.timedelta(days=7) \ No newline at end of file + return timezone.now() > self.created_at + timezone.timedelta(days=7) From d79589294ee79c571ce6fdcd2f8ffc316710660d Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Tue, 14 Jan 2025 16:36:58 +0000 Subject: [PATCH 54/79] feat: add organisation creation, link org project and survey pages --- home/migrations/0001_initial.py | 2 +- .../0002_organisation_description.py | 18 +++ home/mixins.py | 12 ++ home/models.py | 5 +- home/templates/organisation/create.html | 12 ++ home/templates/organisation/organisation.html | 116 ++++++++++++++++++ home/templates/organisation/update.html | 0 home/templates/projects/create.html | 12 ++ home/templates/projects/list.html | 10 +- home/templates/projects/project.html | 110 +++++++++++++++++ home/urls.py | 4 + home/views.py | 105 +++++++++++++++- static/templates/base.html | 2 +- survey/migrations/0001_initial.py | 4 +- .../migrations/0002_alter_survey_project.py | 20 +++ ..._description_alter_survey_survey_config.py | 23 ++++ survey/models.py | 18 +-- survey/templates/survey/create.html | 12 ++ survey/templates/survey/survey.html | 33 +++-- survey/urls.py | 1 + survey/views.py | 23 +++- 21 files changed, 507 insertions(+), 35 deletions(-) create mode 100644 home/migrations/0002_organisation_description.py create mode 100644 home/mixins.py create mode 100644 home/templates/organisation/create.html create mode 100644 home/templates/organisation/organisation.html create mode 100644 home/templates/organisation/update.html create mode 100644 home/templates/projects/create.html create mode 100644 home/templates/projects/project.html create mode 100644 survey/migrations/0002_alter_survey_project.py create mode 100644 survey/migrations/0003_alter_survey_description_alter_survey_survey_config.py create mode 100644 survey/templates/survey/create.html diff --git a/home/migrations/0001_initial.py b/home/migrations/0001_initial.py index b647204..0bf470c 100644 --- a/home/migrations/0001_initial.py +++ b/home/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2025-01-14 11:28 +# Generated by Django 5.1.2 on 2025-01-14 14:06 import django.db.models.deletion from django.conf import settings diff --git a/home/migrations/0002_organisation_description.py b/home/migrations/0002_organisation_description.py new file mode 100644 index 0000000..a90eb5a --- /dev/null +++ b/home/migrations/0002_organisation_description.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2025-01-14 14:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='organisation', + name='description', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/home/mixins.py b/home/mixins.py new file mode 100644 index 0000000..fbac19e --- /dev/null +++ b/home/mixins.py @@ -0,0 +1,12 @@ +from django.shortcuts import redirect +from django.urls import reverse_lazy + +from .models import User, Organisation + +class OrganisationRequiredMixin: + def dispatch(self, request, *args, **kwargs): + if request.user.organisation_set.count() > 0: + return super().dispatch(request, *args, **kwargs) + else: + return redirect("organisation_create") + diff --git a/home/models.py b/home/models.py index afc3edc..820d683 100644 --- a/home/models.py +++ b/home/models.py @@ -4,7 +4,7 @@ BaseUserManager, PermissionsMixin, ) - +from django.urls import reverse class UserManager(BaseUserManager): @@ -53,6 +53,7 @@ def __str__(self): class Organisation(models.Model): name = models.CharField(max_length=200) + description = models.TextField(blank=True, null=True) members = models.ManyToManyField(User, through="OrganisationMembership") created_at = models.DateTimeField(auto_now_add=True) @@ -107,6 +108,8 @@ def user_can_view(self, user): organisation__organisationmembership__user=user ).exists() + def get_absolute_url(self): + return reverse("project", kwargs={"project_id": self.pk}) class ProjectOrganisation(models.Model): project = models.ForeignKey(Project, on_delete=models.CASCADE) diff --git a/home/templates/organisation/create.html b/home/templates/organisation/create.html new file mode 100644 index 0000000..9772ef8 --- /dev/null +++ b/home/templates/organisation/create.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block content %} +
+

Create an organisation

+ +
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} diff --git a/home/templates/organisation/organisation.html b/home/templates/organisation/organisation.html new file mode 100644 index 0000000..c2ab130 --- /dev/null +++ b/home/templates/organisation/organisation.html @@ -0,0 +1,116 @@ +{% extends "base.html" %} +{% load project_filters %} +{% block content %} +
+ +

{{ organisation.name }}

+

{{ organisation.description }}

+
+
+
+

Projects

+ {% if can_create %} + + + + {% endif %} +
+
+ {% for project in projects %} +
+
+
{{ project.name }}
+
+
+ {% for org in project_orgs|get_item:project.id %}{{ org.name }}{% endfor %} +
+ Responses: {{ project.surveys.count }} +
+
+ {% if can_edit|get_item:project.id %} + + + + {% endif %} + + + +
+
+
+ {% empty %} +

No projects found.

+ {% endfor %} +
+ {% comment %} pagination {% endcomment %} + {% if page_obj.has_other_pages %} + + +

Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}

+ {% endif %} +
+ +{% endblock %} diff --git a/home/templates/organisation/update.html b/home/templates/organisation/update.html new file mode 100644 index 0000000..e69de29 diff --git a/home/templates/projects/create.html b/home/templates/projects/create.html new file mode 100644 index 0000000..8d14697 --- /dev/null +++ b/home/templates/projects/create.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} +{% block content %} +
+

Create project

+ +
+ {% csrf_token %} + {{ form.as_p }} + +
+
+{% endblock %} diff --git a/home/templates/projects/list.html b/home/templates/projects/list.html index e9e8bf2..2814f84 100644 --- a/home/templates/projects/list.html +++ b/home/templates/projects/list.html @@ -3,10 +3,10 @@ {% block content %}
-

Studies

+

Projects

{% if can_create %} - + {% endif %}
@@ -27,19 +27,19 @@
{{ project.name }}
{% endif %} - +
{% empty %} -

No studies found.

+

No projects found.

{% endfor %} {% comment %} pagination {% endcomment %} {% if page_obj.has_other_pages %} - +
{% if messages %} {% for message in messages %} {% if message.tags == 'success' %} - + {% endif %} - {% endfor %} - {% endif %} - - {% if messages %} - {% for message in messages %} {% if message.tags == 'error' %} - + {% elif message.tags == 'warning' %} - + {% elif message.tags == 'info' %} - + {% endif %} {% endfor %} {% endif %} - - {% block content %} - - {% endblock %} + {% block content %}{% endblock %}
-
- - - - - + + + + diff --git a/static/templates/login_base.html b/static/templates/login_base.html index 2aa26d0..23ec6f9 100644 --- a/static/templates/login_base.html +++ b/static/templates/login_base.html @@ -1,62 +1,56 @@ {% load static %} {% load django_bootstrap5 %} - - - - - {% block title %}SORT{% endblock %} - - - - - - -
-
-

SORT

-

The Self-Assessment of Organisational Readiness Tool

-
- -
-
- {% if messages %} - {% for message in messages %} - {% if message.tags == 'success' %} - - {% endif %} - {% endfor %} - {% endif %} - - {% if messages %} - {% for message in messages %} - {% if message.tags == 'error' %} - - {% elif message.tags == 'warning' %} - - {% elif message.tags == 'info' %} - - {% endif %} - {% endfor %} - {% endif %} - - {% block content %} - - {% endblock %} + + + + + {% block title %}SORT{% endblock %} + + + + + + +
+
+

SORT

+

The Self-Assessment of Organisational Readiness Tool

+
+
+
+ {% if messages %} + {% for message in messages %} + {% if message.tags == 'success' %} + + {% endif %} + {% if message.tags == 'error' %} + + {% elif message.tags == 'warning' %} + + {% elif message.tags == 'info' %} + + {% endif %} + {% endfor %} + {% endif %} + {% block content %}{% endblock %} +
-
- - + diff --git a/survey/templates/survey/survey.html b/survey/templates/survey/survey.html index b9297cd..8710069 100644 --- a/survey/templates/survey/survey.html +++ b/survey/templates/survey/survey.html @@ -1,25 +1,38 @@ {% extends 'base.html' %} - {% block content %} +
+ +

{{ survey.name }}

{{ survey.description }}

- -
-

Responses

+

Responses

    - {% for survey_response in survey.survey_response.all %} -
  • {{ survey_response.answers }}
    • - {% endfor %} + {% for survey_response in survey.survey_response.all %}
  • {{ survey_response.answers }}
    • {% endfor %}
From e3d04985d2e52ac61bc1d060a59f06f2900c9d61 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Wed, 15 Jan 2025 18:03:02 +0000 Subject: [PATCH 61/79] chore: remove projects page --- home/templates/projects/project.html | 8 +++-- home/urls.py | 1 - home/views.py | 46 ---------------------------- 3 files changed, 6 insertions(+), 49 deletions(-) diff --git a/home/templates/projects/project.html b/home/templates/projects/project.html index 75cae12..7e213f9 100644 --- a/home/templates/projects/project.html +++ b/home/templates/projects/project.html @@ -21,7 +21,9 @@

{{ project.name }}

Surveys

{% if can_create %} - + {% endif %}
@@ -35,7 +37,9 @@
{{ survey.name }}
- +
diff --git a/home/urls.py b/home/urls.py index 1151413..3ec7a92 100644 --- a/home/urls.py +++ b/home/urls.py @@ -34,7 +34,6 @@ # path('password_reset/expired/', views.PasswordResetExpiredView.as_view(), name='password_reset_expired'), path("myorganisation/", views.MyOrganisationView.as_view(), name="myorganisation"), path("organisation/create/", views.OrganisationCreateView.as_view(), name="organisation_create"), - path("projects/", views.ProjectListView.as_view(), name="projects"), # path("projects/create/", views.ProjectCreateView.as_view(), name="project_create"), path("projects//", views.ProjectView.as_view(), name="project"), path("projects/create//", views.ProjectCreateView.as_view(), name="project_create"), diff --git a/home/views.py b/home/views.py index daada9b..adfa25e 100644 --- a/home/views.py +++ b/home/views.py @@ -175,52 +175,6 @@ def form_valid(self, form): return redirect("myorganisation") -class ProjectListView(LoginRequiredMixin, ListView): - model = Project - template_name = "projects/list.html" - context_object_name = "projects" - paginate_by = 10 - - def get_queryset(self): - # Get all projects associated with user's organisations - projects = ( - Project.objects.filter( - organisations__organisationmembership__user=self.request.user - ) - .distinct() - .select_related("created_by") - .prefetch_related( - "organisations", "organisations__organisationmembership_set" - ) - ) - - return projects - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - # Add edit permissions for each project - context["can_edit"] = { - project.id: project.user_can_edit(self.request.user) - for project in context["projects"] - } - context["can_create"] = OrganisationMembership.objects.filter( - user=self.request.user, role="ADMIN" - ).exists() - - user_orgs = set( - OrganisationMembership.objects.filter(user=self.request.user).values_list( - "organisation_id", flat=True - ) - ) - - context["project_orgs"] = { - project.id: [ - org for org in project.organisations.all() if org.id in user_orgs - ] - for project in context["projects"] - } - return context - class ProjectView(LoginRequiredMixin, ListView): template_name = "projects/project.html" From f94217047ea980e544e6acd803ddfde1cbe1b4b7 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Wed, 15 Jan 2025 19:41:27 +0000 Subject: [PATCH 62/79] refactor: use permissions and services to structure the code Add test data. --- data/001_users.json | 40 + data/002_organisations.json | 20 + data/003_memberships.json | 32 + data/004_projects.json | 20 + data/005_project_organisations.json | 22 + data/006_surveys.json | 50 + data/007_survey_responses.json | 24 + data/README.md | 9 + data/{ => questions}/questionnaires.json | 114 +-- data/{ => questions}/questions.json | 938 +++++++++--------- home/constants.py | 19 + .../commands/create_org_proj_test_data.py | 42 - ...ationmembership_role_guestprojectaccess.py | 35 + home/models.py | 50 +- home/permissions.py | 59 ++ home/services.py | 34 + home/views.py | 32 +- static/templates/base.html | 16 +- .../0004_alter_surveyresponse_survey.py | 19 + 19 files changed, 964 insertions(+), 611 deletions(-) create mode 100644 data/001_users.json create mode 100644 data/002_organisations.json create mode 100644 data/003_memberships.json create mode 100644 data/004_projects.json create mode 100644 data/005_project_organisations.json create mode 100644 data/006_surveys.json create mode 100644 data/007_survey_responses.json create mode 100644 data/README.md rename data/{ => questions}/questionnaires.json (97%) rename data/{ => questions}/questions.json (96%) create mode 100644 home/constants.py delete mode 100644 home/management/commands/create_org_proj_test_data.py create mode 100644 home/migrations/0003_alter_organisationmembership_role_guestprojectaccess.py create mode 100644 home/permissions.py create mode 100644 home/services.py create mode 100644 survey/migrations/0004_alter_surveyresponse_survey.py diff --git a/data/001_users.json b/data/001_users.json new file mode 100644 index 0000000..9da8214 --- /dev/null +++ b/data/001_users.json @@ -0,0 +1,40 @@ +[ + { + "model": "home.user", + "pk": 1, + "fields": { + "email": "admin@test.com", + "first_name": "Admin", + "last_name": "User", + "password": "pbkdf2_sha256$870000$generate something random$jxrzQIBcrwqeZdLY5hJxM5tugSuSyGIDAsVVx2qIWBY=", + "is_staff": true, + "is_superuser": true, + "is_active": true, + "date_joined": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.user", + "pk": 2, + "fields": { + "email": "member@test.com", + "first_name": "Member", + "last_name": "User", + "password": "pbkdf2_sha256$870000$generate something random$so3lHdCP3pMjsJw0HoSAlZHqIIFRQqYU7C05ndgP/i4=", + "is_active": true, + "date_joined": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.user", + "pk": 3, + "fields": { + "email": "guest@test.com", + "first_name": "Guest", + "last_name": "User", + "password": "pbkdf2_sha256$870000$generate something random$bVomqYDHpEn8jXJdd8FGdl5Ju+/kjBm5//jAJkICnEU=", + "is_active": true, + "date_joined": "2024-01-15T00:00:00Z" + } + } +] \ No newline at end of file diff --git a/data/002_organisations.json b/data/002_organisations.json new file mode 100644 index 0000000..00b2216 --- /dev/null +++ b/data/002_organisations.json @@ -0,0 +1,20 @@ +[ + { + "model": "home.organisation", + "pk": 1, + "fields": { + "name": "RSE Team", + "description": "Research Software Engineering Team", + "created_at": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.organisation", + "pk": 2, + "fields": { + "name": "Research Group", + "description": "Academic Research Group", + "created_at": "2024-01-15T00:00:00Z" + } + } +] \ No newline at end of file diff --git a/data/003_memberships.json b/data/003_memberships.json new file mode 100644 index 0000000..eaa69cb --- /dev/null +++ b/data/003_memberships.json @@ -0,0 +1,32 @@ +[ + { + "model": "home.organisationmembership", + "pk": 1, + "fields": { + "user": 1, + "organisation": 1, + "role": "ADMIN", + "joined_at": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.organisationmembership", + "pk": 2, + "fields": { + "user": 2, + "organisation": 1, + "role": "MEMBER", + "joined_at": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.organisationmembership", + "pk": 3, + "fields": { + "user": 3, + "organisation": 1, + "role": "GUEST", + "joined_at": "2024-01-15T00:00:00Z" + } + } +] \ No newline at end of file diff --git a/data/004_projects.json b/data/004_projects.json new file mode 100644 index 0000000..45bc5ec --- /dev/null +++ b/data/004_projects.json @@ -0,0 +1,20 @@ +[ + { + "model": "home.project", + "pk": 1, + "fields": { + "name": "DataVis 2025", + "created_by": 1, + "created_on": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.project", + "pk": 2, + "fields": { + "name": "Data Analysis Platform", + "created_by": 1, + "created_on": "2024-01-15T00:00:00Z" + } + } +] \ No newline at end of file diff --git a/data/005_project_organisations.json b/data/005_project_organisations.json new file mode 100644 index 0000000..9fe90c4 --- /dev/null +++ b/data/005_project_organisations.json @@ -0,0 +1,22 @@ +[ + { + "model": "home.projectorganisation", + "pk": 1, + "fields": { + "project": 1, + "organisation": 1, + "added_by": 1, + "added_at": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.projectorganisation", + "pk": 2, + "fields": { + "project": 1, + "organisation": 2, + "added_by": 1, + "added_at": "2024-01-15T00:00:00Z" + } + } +] diff --git a/data/006_surveys.json b/data/006_surveys.json new file mode 100644 index 0000000..4d45b14 --- /dev/null +++ b/data/006_surveys.json @@ -0,0 +1,50 @@ +[ + { + "model": "survey.survey", + "pk": 1, + "fields": { + "name": "DataVis 2025 - Satisfaction Survey", + "description": "Help us improve our services", + "survey_config": { + "questions": [ + { + "id": "q1", + "type": "rating", + "question": "How satisfied are you with the service?", + "options": ["1", "2", "3", "4", "5"] + }, + { + "id": "q2", + "type": "text", + "question": "What could we improve?" + } + ] + }, + "project": 1 + } + }, + { + "model": "survey.survey", + "pk": 2, + "fields": { + "name": "DataVis 2025 - Feature Feedback", + "description": "Tell us about your feature usage", + "survey_config": { + "questions": [ + { + "id": "q1", + "type": "multiple_choice", + "question": "Which features do you use most?", + "options": ["Analysis", "Visualization", "Export", "Sharing"] + }, + { + "id": "q2", + "type": "text", + "question": "Any additional comments?" + } + ] + }, + "project": 1 + } + } +] \ No newline at end of file diff --git a/data/007_survey_responses.json b/data/007_survey_responses.json new file mode 100644 index 0000000..dcec430 --- /dev/null +++ b/data/007_survey_responses.json @@ -0,0 +1,24 @@ +[ + { + "model": "survey.surveyresponse", + "pk": 1, + "fields": { + "survey": 1, + "answers": { + "q1": "4", + "q2": "The interface could be more intuitive" + } + } + }, + { + "model": "survey.surveyresponse", + "pk": 2, + "fields": { + "survey": 1, + "answers": { + "q1": "5", + "q2": "Everything works great!" + } + } + } +] \ No newline at end of file diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..ea48af6 --- /dev/null +++ b/data/README.md @@ -0,0 +1,9 @@ +# Import test data + +Run: + +```bash +python manage.py loaddata ./data/001_users.json ./data/002_organisations.json ./data/003_memberships.json ./data/004_projects.json ./data/005_project_organisations.json ./data/006_surveys.json ./data/007_survey_responses.json +``` + +The password for users is their respective role name in lowercases. diff --git a/data/questionnaires.json b/data/questions/questionnaires.json similarity index 97% rename from data/questionnaires.json rename to data/questions/questionnaires.json index aac96bc..18a5798 100644 --- a/data/questionnaires.json +++ b/data/questions/questionnaires.json @@ -1,58 +1,58 @@ -[ - { - "model": "survey.questionnaire", - "pk": 3, - "fields": { - "title": "Consent", - "description": "Before you begin, we kindly ask you to review the information below regarding your participation in this questionnaire. It is important that you understand the purpose of this study, what your involvement entails, and how your data will be used. By completing and submitting your responses, you are agreeing to participate voluntarily and provide consent for the use of your data in accordance with the terms outlined. In completing and submitting my responses to this questionnaire, I understand: Please tick the checkboxes." - } - }, - { - "model": "survey.questionnaire", - "pk": 4, - "fields": { - "title": "A. Releasing Potential", - "description": "This includes spotting talent, enthusiasm, and resilience, �learning by doing�, training and research skill use in practice. It also covers understanding the importance of an enabling environment, facilitating and supporting research careers from the start." - } - }, - { - "model": "survey.questionnaire", - "pk": 5, - "fields": { - "title": "B. Embedding research", - "description": "This includes reducing barriers to research related activities by providing time and resources. It covers making research legitimate in the organisation, recognising the impact of research and nurses' contribution to research." - } - }, - { - "model": "survey.questionnaire", - "pk": 6, - "fields": { - "title": "C. Linkages and Leadership", - "description": "This includes activities related to forming research links outside the organisation, promoting nurse research leadership to influence the wider research agenda." - } - }, - { - "model": "survey.questionnaire", - "pk": 7, - "fields": { - "title": "D. Inclusive research delivery", - "description": "This includes activities related supporting the public and patient�s involvement in research. It also covers engaging the wider nursing workforce in delivering portfolio research, creating more opportunities to deliver research and making the contribution of nurses visible." - } - }, - { - "model": "survey.questionnaire", - "pk": 8, - "fields": { - "title": "E. Digital enabled research", - "description": "It covers activities related to research leadership and skills in digital technologies and data science including the skills needed to undertake research and service developments." - } - }, - { - "model": "survey.questionnaire", - "pk": 9, - "fields": { - "title": "Demographic Information", - "description": "Please provide us with this information." - } - } +[ + { + "model": "survey.questionnaire", + "pk": 3, + "fields": { + "title": "Consent", + "description": "Before you begin, we kindly ask you to review the information below regarding your participation in this questionnaire. It is important that you understand the purpose of this study, what your involvement entails, and how your data will be used. By completing and submitting your responses, you are agreeing to participate voluntarily and provide consent for the use of your data in accordance with the terms outlined. In completing and submitting my responses to this questionnaire, I understand: Please tick the checkboxes." + } + }, + { + "model": "survey.questionnaire", + "pk": 4, + "fields": { + "title": "A. Releasing Potential", + "description": "This includes spotting talent, enthusiasm, and resilience, �learning by doing�, training and research skill use in practice. It also covers understanding the importance of an enabling environment, facilitating and supporting research careers from the start." + } + }, + { + "model": "survey.questionnaire", + "pk": 5, + "fields": { + "title": "B. Embedding research", + "description": "This includes reducing barriers to research related activities by providing time and resources. It covers making research legitimate in the organisation, recognising the impact of research and nurses' contribution to research." + } + }, + { + "model": "survey.questionnaire", + "pk": 6, + "fields": { + "title": "C. Linkages and Leadership", + "description": "This includes activities related to forming research links outside the organisation, promoting nurse research leadership to influence the wider research agenda." + } + }, + { + "model": "survey.questionnaire", + "pk": 7, + "fields": { + "title": "D. Inclusive research delivery", + "description": "This includes activities related supporting the public and patient�s involvement in research. It also covers engaging the wider nursing workforce in delivering portfolio research, creating more opportunities to deliver research and making the contribution of nurses visible." + } + }, + { + "model": "survey.questionnaire", + "pk": 8, + "fields": { + "title": "E. Digital enabled research", + "description": "It covers activities related to research leadership and skills in digital technologies and data science including the skills needed to undertake research and service developments." + } + }, + { + "model": "survey.questionnaire", + "pk": 9, + "fields": { + "title": "Demographic Information", + "description": "Please provide us with this information." + } + } ] \ No newline at end of file diff --git a/data/questions.json b/data/questions/questions.json similarity index 96% rename from data/questions.json rename to data/questions/questions.json index 96af8b2..2cfdeb1 100644 --- a/data/questions.json +++ b/data/questions/questions.json @@ -1,470 +1,470 @@ -[ - { - "model": "survey.question", - "pk": 7, - "fields": { - "questionnaire": 3, - "question_text": "My participation is voluntary", - "question_type": "boolean" - } - }, - { - "model": "survey.question", - "pk": 8, - "fields": { - "questionnaire": 3, - "question_text": "All responses will be anonymous", - "question_type": "boolean" - } - }, - { - "model": "survey.question", - "pk": 9, - "fields": { - "questionnaire": 3, - "question_text": "I can withdraw during the online completion but once the answers are submitted, it will no longer be possible to withdraw due to the anonymous data collection procedure.", - "question_type": "boolean" - } - }, - { - "model": "survey.question", - "pk": 10, - "fields": { - "questionnaire": 3, - "question_text": "The data will be used for research purposes and will be published but that my identity will not be revealed.", - "question_type": "boolean" - } - }, - { - "model": "survey.question", - "pk": 11, - "fields": { - "questionnaire": 4, - "question_text": "A1. has a system to talent spot and support individuals who are active in-service development/ QI to progress on to research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 12, - "fields": { - "questionnaire": 4, - "question_text": "A2. has research role models and named nursing research leaders", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 13, - "fields": { - "questionnaire": 4, - "question_text": "A3. identifies and celebrates success in the nursing research related activity", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 14, - "fields": { - "questionnaire": 4, - "question_text": "A4. provides research advice sessions where nurses can explore ideas for project development", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 15, - "fields": { - "questionnaire": 4, - "question_text": "A5. provides help to nurses to navigate research funding submissions, ethics and governance systems", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 16, - "fields": { - "questionnaire": 4, - "question_text": "A6. has a finance department that can cost research project involvement for external funding applications", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 17, - "fields": { - "questionnaire": 4, - "question_text": "A7. has an active research-related mentorship programme", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 18, - "fields": { - "questionnaire": 4, - "question_text": "A8. provides mentorship to nurses to successfully apply for internships and fellowship opportunities", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 19, - "fields": { - "questionnaire": 4, - "question_text": "A9. enables the use of awarded grant funding in the manner intended (for example, protected time and spending decisions)", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 20, - "fields": { - "questionnaire": 4, - "question_text": "A10. provides nurses with access to research learning opportunities to develop research skills delivered through our R&D department, service development, education, or training departments", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 21, - "fields": { - "questionnaire": 4, - "question_text": "A11. provides nurses with access to experts who can advise on developing project proposals", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 22, - "fields": { - "questionnaire": 4, - "question_text": "A12. has a mission statement that includes an ambition to do research as a core activity", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 23, - "fields": { - "questionnaire": 4, - "question_text": "A13. has a strategic document to support research capacity development for nursing", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 24, - "fields": { - "questionnaire": 4, - "question_text": "A14. has a research capacity delivery plan which aims to maximise the use, delivery, collaboration and leadership in nursing research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 25, - "fields": { - "questionnaire": 4, - "question_text": "A15. has a dedicated database of projects that are nurse led or where nurses have contributed", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 26, - "fields": { - "questionnaire": 4, - "question_text": "A16. monitors research supervision and successful project development and delivery", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 27, - "fields": { - "questionnaire": 4, - "question_text": "A17. develops good news stories of research in our internal and external communications", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 28, - "fields": { - "questionnaire": 4, - "question_text": "A18. includes research in our induction process", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 29, - "fields": { - "questionnaire": 4, - "question_text": "A19. offers pre-registration research placement opportunities", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 30, - "fields": { - "questionnaire": 4, - "question_text": "A20. offers continuing professional development in research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 31, - "fields": { - "questionnaire": 4, - "question_text": "A21. provides opportunities to use research skills and experience of leadership at post-masters", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 32, - "fields": { - "questionnaire": 4, - "question_text": "A22. provides opportunities to use research skills and experience of leadership at post-doctoral levels.", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 33, - "fields": { - "questionnaire": 4, - "question_text": "A23. If you would like to add any comments with regards to any of the statements in section above (Releasing Potential), please write below.", - "question_type": "text" - } - }, - { - "model": "survey.question", - "pk": 34, - "fields": { - "questionnaire": 5, - "question_text": "B1. provides protected time for clinical nurses to support research related activities", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 35, - "fields": { - "questionnaire": 5, - "question_text": "B2. provides resources (for example, time and/or funds) to support Public and Patient Involvement and Engagement to identify and develop research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 36, - "fields": { - "questionnaire": 5, - "question_text": "B3. develops impact stories from projects where nurses support, participate or lead research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 37, - "fields": { - "questionnaire": 5, - "question_text": "B4. collects case studies of where Public and Patient Involvement and Engagement has made a difference to research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 38, - "fields": { - "questionnaire": 5, - "question_text": "B5. actively communicates to the nursing workforce, clinical managers and executive team about how the involvement of nurses in research has made a difference to services and people", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 39, - "fields": { - "questionnaire": 5, - "question_text": "B6. If you would like to add any comments with regards to any of the statements in the section above (Embedding Research), please write below.", - "question_type": "text" - } - }, - { - "model": "survey.question", - "pk": 40, - "fields": { - "questionnaire": 6, - "question_text": "C1. take part in research leadership and advisory activities outside our organisation (for example, sitting on ethics committees; funding committees; editorial boards and reviewing papers)", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 41, - "fields": { - "questionnaire": 6, - "question_text": "C2. take part in research leadership and advisory activities outside organisations", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 42, - "fields": { - "questionnaire": 6, - "question_text": "C3. work with professional bodies and national and regional policy structures to influence the research agenda", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 43, - "fields": { - "questionnaire": 6, - "question_text": "C4. are members of forums outside our organisation which support research activity", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 44, - "fields": { - "questionnaire": 6, - "question_text": "C5. If you would like to add any comments with regards to any of the statements in the section above (Linkages and Leadership), please write below.", - "question_type": "text" - } - }, - { - "model": "survey.question", - "pk": 45, - "fields": { - "questionnaire": 7, - "question_text": "D1. have skills to support Public and Patient Involvement and Engagement", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 46, - "fields": { - "questionnaire": 7, - "question_text": "D2. use their expertise to deliver research, including portfolio (commercial and non-commercial) research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 47, - "fields": { - "questionnaire": 7, - "question_text": "D3. who deliver research have their contribution recognised in research outputs (for example, through acknowledgement, co-authorship)", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 48, - "fields": { - "questionnaire": 7, - "question_text": "D4. who work at an advanced, specialist, and consultant levels of practice act as principal investigators (PIs) for portfolio research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 49, - "fields": { - "questionnaire": 7, - "question_text": "D5. If you would like to add any comments with regards to any of the statements in the section above (Inclusive research delivery), please write below.", - "question_type": "text" - } - }, - { - "model": "survey.question", - "pk": 50, - "fields": { - "questionnaire": 8, - "question_text": "E1. provides training for nurses to enable them to practise effectively in a digitally enabled environment", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 51, - "fields": { - "questionnaire": 8, - "question_text": "E2. trains nurses to use and interpret data to make improvements to care (using audit, service evaluation or research)", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 52, - "fields": { - "questionnaire": 8, - "question_text": "E4. has digital nurse leaders in place who can provide advice and guidance in the use of digital technology in research", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 53, - "fields": { - "questionnaire": 8, - "question_text": "E3. has digital nurse leaders in place who can provide advice and guidance in the use of digital technology in service development", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 54, - "fields": { - "questionnaire": 8, - "question_text": "E5. has the infrastructure to support visualisation of data using business intelligence tools", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 55, - "fields": { - "questionnaire": 8, - "question_text": "E6. has the internal structures that facilitate, support and enable nurse-led digital innovation", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 56, - "fields": { - "questionnaire": 8, - "question_text": "E7. has effective partnerships with technology suppliers to support digital developments and innovation that meet the needs of nurses", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 57, - "fields": { - "questionnaire": 8, - "question_text": "E8. collects and shares case study examples of improvements to care using digital technology", - "question_type": "rating" - } - }, - { - "model": "survey.question", - "pk": 58, - "fields": { - "questionnaire": 8, - "question_text": "E9. collects and shares case study examples of improvements to research using digital technology", - "question_type": "rating" - } - } +[ + { + "model": "survey.question", + "pk": 7, + "fields": { + "questionnaire": 3, + "question_text": "My participation is voluntary", + "question_type": "boolean" + } + }, + { + "model": "survey.question", + "pk": 8, + "fields": { + "questionnaire": 3, + "question_text": "All responses will be anonymous", + "question_type": "boolean" + } + }, + { + "model": "survey.question", + "pk": 9, + "fields": { + "questionnaire": 3, + "question_text": "I can withdraw during the online completion but once the answers are submitted, it will no longer be possible to withdraw due to the anonymous data collection procedure.", + "question_type": "boolean" + } + }, + { + "model": "survey.question", + "pk": 10, + "fields": { + "questionnaire": 3, + "question_text": "The data will be used for research purposes and will be published but that my identity will not be revealed.", + "question_type": "boolean" + } + }, + { + "model": "survey.question", + "pk": 11, + "fields": { + "questionnaire": 4, + "question_text": "A1. has a system to talent spot and support individuals who are active in-service development/ QI to progress on to research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 12, + "fields": { + "questionnaire": 4, + "question_text": "A2. has research role models and named nursing research leaders", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 13, + "fields": { + "questionnaire": 4, + "question_text": "A3. identifies and celebrates success in the nursing research related activity", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 14, + "fields": { + "questionnaire": 4, + "question_text": "A4. provides research advice sessions where nurses can explore ideas for project development", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 15, + "fields": { + "questionnaire": 4, + "question_text": "A5. provides help to nurses to navigate research funding submissions, ethics and governance systems", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 16, + "fields": { + "questionnaire": 4, + "question_text": "A6. has a finance department that can cost research project involvement for external funding applications", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 17, + "fields": { + "questionnaire": 4, + "question_text": "A7. has an active research-related mentorship programme", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 18, + "fields": { + "questionnaire": 4, + "question_text": "A8. provides mentorship to nurses to successfully apply for internships and fellowship opportunities", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 19, + "fields": { + "questionnaire": 4, + "question_text": "A9. enables the use of awarded grant funding in the manner intended (for example, protected time and spending decisions)", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 20, + "fields": { + "questionnaire": 4, + "question_text": "A10. provides nurses with access to research learning opportunities to develop research skills delivered through our R&D department, service development, education, or training departments", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 21, + "fields": { + "questionnaire": 4, + "question_text": "A11. provides nurses with access to experts who can advise on developing project proposals", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 22, + "fields": { + "questionnaire": 4, + "question_text": "A12. has a mission statement that includes an ambition to do research as a core activity", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 23, + "fields": { + "questionnaire": 4, + "question_text": "A13. has a strategic document to support research capacity development for nursing", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 24, + "fields": { + "questionnaire": 4, + "question_text": "A14. has a research capacity delivery plan which aims to maximise the use, delivery, collaboration and leadership in nursing research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 25, + "fields": { + "questionnaire": 4, + "question_text": "A15. has a dedicated database of projects that are nurse led or where nurses have contributed", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 26, + "fields": { + "questionnaire": 4, + "question_text": "A16. monitors research supervision and successful project development and delivery", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 27, + "fields": { + "questionnaire": 4, + "question_text": "A17. develops good news stories of research in our internal and external communications", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 28, + "fields": { + "questionnaire": 4, + "question_text": "A18. includes research in our induction process", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 29, + "fields": { + "questionnaire": 4, + "question_text": "A19. offers pre-registration research placement opportunities", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 30, + "fields": { + "questionnaire": 4, + "question_text": "A20. offers continuing professional development in research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 31, + "fields": { + "questionnaire": 4, + "question_text": "A21. provides opportunities to use research skills and experience of leadership at post-masters", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 32, + "fields": { + "questionnaire": 4, + "question_text": "A22. provides opportunities to use research skills and experience of leadership at post-doctoral levels.", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 33, + "fields": { + "questionnaire": 4, + "question_text": "A23. If you would like to add any comments with regards to any of the statements in section above (Releasing Potential), please write below.", + "question_type": "text" + } + }, + { + "model": "survey.question", + "pk": 34, + "fields": { + "questionnaire": 5, + "question_text": "B1. provides protected time for clinical nurses to support research related activities", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 35, + "fields": { + "questionnaire": 5, + "question_text": "B2. provides resources (for example, time and/or funds) to support Public and Patient Involvement and Engagement to identify and develop research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 36, + "fields": { + "questionnaire": 5, + "question_text": "B3. develops impact stories from projects where nurses support, participate or lead research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 37, + "fields": { + "questionnaire": 5, + "question_text": "B4. collects case studies of where Public and Patient Involvement and Engagement has made a difference to research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 38, + "fields": { + "questionnaire": 5, + "question_text": "B5. actively communicates to the nursing workforce, clinical managers and executive team about how the involvement of nurses in research has made a difference to services and people", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 39, + "fields": { + "questionnaire": 5, + "question_text": "B6. If you would like to add any comments with regards to any of the statements in the section above (Embedding Research), please write below.", + "question_type": "text" + } + }, + { + "model": "survey.question", + "pk": 40, + "fields": { + "questionnaire": 6, + "question_text": "C1. take part in research leadership and advisory activities outside our organisation (for example, sitting on ethics committees; funding committees; editorial boards and reviewing papers)", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 41, + "fields": { + "questionnaire": 6, + "question_text": "C2. take part in research leadership and advisory activities outside organisations", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 42, + "fields": { + "questionnaire": 6, + "question_text": "C3. work with professional bodies and national and regional policy structures to influence the research agenda", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 43, + "fields": { + "questionnaire": 6, + "question_text": "C4. are members of forums outside our organisation which support research activity", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 44, + "fields": { + "questionnaire": 6, + "question_text": "C5. If you would like to add any comments with regards to any of the statements in the section above (Linkages and Leadership), please write below.", + "question_type": "text" + } + }, + { + "model": "survey.question", + "pk": 45, + "fields": { + "questionnaire": 7, + "question_text": "D1. have skills to support Public and Patient Involvement and Engagement", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 46, + "fields": { + "questionnaire": 7, + "question_text": "D2. use their expertise to deliver research, including portfolio (commercial and non-commercial) research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 47, + "fields": { + "questionnaire": 7, + "question_text": "D3. who deliver research have their contribution recognised in research outputs (for example, through acknowledgement, co-authorship)", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 48, + "fields": { + "questionnaire": 7, + "question_text": "D4. who work at an advanced, specialist, and consultant levels of practice act as principal investigators (PIs) for portfolio research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 49, + "fields": { + "questionnaire": 7, + "question_text": "D5. If you would like to add any comments with regards to any of the statements in the section above (Inclusive research delivery), please write below.", + "question_type": "text" + } + }, + { + "model": "survey.question", + "pk": 50, + "fields": { + "questionnaire": 8, + "question_text": "E1. provides training for nurses to enable them to practise effectively in a digitally enabled environment", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 51, + "fields": { + "questionnaire": 8, + "question_text": "E2. trains nurses to use and interpret data to make improvements to care (using audit, service evaluation or research)", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 52, + "fields": { + "questionnaire": 8, + "question_text": "E4. has digital nurse leaders in place who can provide advice and guidance in the use of digital technology in research", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 53, + "fields": { + "questionnaire": 8, + "question_text": "E3. has digital nurse leaders in place who can provide advice and guidance in the use of digital technology in service development", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 54, + "fields": { + "questionnaire": 8, + "question_text": "E5. has the infrastructure to support visualisation of data using business intelligence tools", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 55, + "fields": { + "questionnaire": 8, + "question_text": "E6. has the internal structures that facilitate, support and enable nurse-led digital innovation", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 56, + "fields": { + "questionnaire": 8, + "question_text": "E7. has effective partnerships with technology suppliers to support digital developments and innovation that meet the needs of nurses", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 57, + "fields": { + "questionnaire": 8, + "question_text": "E8. collects and shares case study examples of improvements to care using digital technology", + "question_type": "rating" + } + }, + { + "model": "survey.question", + "pk": 58, + "fields": { + "questionnaire": 8, + "question_text": "E9. collects and shares case study examples of improvements to research using digital technology", + "question_type": "rating" + } + } ] \ No newline at end of file diff --git a/home/constants.py b/home/constants.py new file mode 100644 index 0000000..2cbeeb3 --- /dev/null +++ b/home/constants.py @@ -0,0 +1,19 @@ +from typing import Literal + +ROLE_ADMIN = "ADMIN" +ROLE_MEMBER = "MEMBER" +ROLE_GUEST = "GUEST" + +ROLES = [ + (ROLE_ADMIN, "Admin"), + (ROLE_MEMBER, "Member"), + (ROLE_GUEST, "Guest"), +] +""" +ADMIN: Full control +MEMBER: Can view and edit projects +GUEST: Can view certain projects +""" + + +RoleType = Literal["ADMIN", "MEMBER", "GUEST"] \ No newline at end of file diff --git a/home/management/commands/create_org_proj_test_data.py b/home/management/commands/create_org_proj_test_data.py deleted file mode 100644 index 5861aa7..0000000 --- a/home/management/commands/create_org_proj_test_data.py +++ /dev/null @@ -1,42 +0,0 @@ -from django.core.management.base import BaseCommand -from django.contrib.auth.hashers import make_password -from home.models import ( - User, - Organisation, - OrganisationMembership, - Project, - ProjectOrganisation, -) - - -class Command(BaseCommand): - help = "Creates Organisation and Project test data" - - def handle(self, *args, **kwargs): - # Create users - admin = User.objects.create( - email="admin@admin.com", - password=make_password("admin"), - first_name="Test-admin", - last_name="User", - is_staff=True, - is_superuser=True, - ) - - # Create organisations - org = Organisation.objects.create(name="Datavis Team") - - # Create memberships - OrganisationMembership.objects.create( - user=admin, organisation=org, role="ADMIN" - ) - - # Create projects - project = Project.objects.create(name="Data Surveys 2025", created_by=admin) - - # Create project-org relationships - ProjectOrganisation.objects.create( - project=project, organisation=org, added_by=admin - ) - - self.stdout.write(self.style.SUCCESS("Successfully created Organisation and Project test data")) diff --git a/home/migrations/0003_alter_organisationmembership_role_guestprojectaccess.py b/home/migrations/0003_alter_organisationmembership_role_guestprojectaccess.py new file mode 100644 index 0000000..4bf6cb9 --- /dev/null +++ b/home/migrations/0003_alter_organisationmembership_role_guestprojectaccess.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.2 on 2025-01-15 18:59 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0002_organisation_description'), + ] + + operations = [ + migrations.AlterField( + model_name='organisationmembership', + name='role', + field=models.CharField(choices=[('ADMIN', 'Admin'), ('MEMBER', 'Member'), ('GUEST', 'Guest')], default='GUEST', max_length=20), + ), + migrations.CreateModel( + name='GuestProjectAccess', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('permission', models.CharField(choices=[('VIEW', 'View Only'), ('EDIT', 'View and Edit')], default='VIEW', max_length=10)), + ('granted_at', models.DateTimeField(auto_now_add=True)), + ('granted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='granted_access', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.project')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name_plural': 'Guest project access', + 'unique_together': {('user', 'project')}, + }, + ), + ] diff --git a/home/models.py b/home/models.py index 820d683..5e24b8d 100644 --- a/home/models.py +++ b/home/models.py @@ -5,6 +5,7 @@ PermissionsMixin, ) from django.urls import reverse +from .constants import ROLES, ROLE_ADMIN, ROLE_MEMBER, ROLE_GUEST, RoleType class UserManager(BaseUserManager): @@ -66,26 +67,18 @@ def get_user_role(self, user): class OrganisationMembership(models.Model): - ROLE_CHOICES = [ - ("ADMIN", "Administrator"), - ("MEMBER", "Member"), - ("GUEST", "Guest"), - ] - """ - ADMIN: Full control - MEMBER: Can view and edit projects - GUEST: Can view certain projects - """ + ROLE_CHOICES = ROLES user = models.ForeignKey(User, on_delete=models.CASCADE) organisation = models.ForeignKey(Organisation, on_delete=models.CASCADE) - role = models.CharField(max_length=20, choices=ROLE_CHOICES, default="GUEST") + role = models.CharField(max_length=20, choices=ROLES, default=ROLE_GUEST) joined_at = models.DateTimeField(auto_now_add=True) class Meta: unique_together = ["user", "organisation"] """ A user can only be a member of an organisation once """ + class Project(models.Model): name = models.CharField(max_length=100) organisations = models.ManyToManyField(Organisation, through="ProjectOrganisation") @@ -96,21 +89,13 @@ class Project(models.Model): def __str__(self): return self.name - def user_can_edit(self, user): - user_orgs = self.projectorganisation_set.filter( - organisation__organisationmembership__user=user, - organisation__organisationmembership__role__in=["ADMIN", "MEMBER"], - ) - return user_orgs.exists() - - def user_can_view(self, user): - return self.projectorganisation_set.filter( - organisation__organisationmembership__user=user - ).exists() + def get_guest_users(self): + return GuestProjectAccess.objects.filter(project=self).select_related("user") def get_absolute_url(self): return reverse("project", kwargs={"project_id": self.pk}) + class ProjectOrganisation(models.Model): project = models.ForeignKey(Project, on_delete=models.CASCADE) organisation = models.ForeignKey(Organisation, on_delete=models.CASCADE) @@ -119,3 +104,24 @@ class ProjectOrganisation(models.Model): class Meta: unique_together = ["project", "organisation"] + + +class GuestProjectAccess(models.Model): + PERMISSION_CHOICES = [ + ("VIEW", "View Only"), + ("EDIT", "View and Edit"), + ] + + user = models.ForeignKey(User, on_delete=models.CASCADE) + project = models.ForeignKey(Project, on_delete=models.CASCADE) + permission = models.CharField( + max_length=10, choices=PERMISSION_CHOICES, default="VIEW" + ) + granted_by = models.ForeignKey( + User, on_delete=models.SET_NULL, null=True, related_name="granted_access" + ) + granted_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = ["user", "project"] + verbose_name_plural = "Guest project access" diff --git a/home/permissions.py b/home/permissions.py new file mode 100644 index 0000000..bc1cd5f --- /dev/null +++ b/home/permissions.py @@ -0,0 +1,59 @@ +from typing import Optional +from .models import GuestProjectAccess, Project, User +from .constants import ROLE_ADMIN, ROLE_MEMBER, ROLE_GUEST, RoleType + + +def get_user_role_in_project(user: User, project: Project) -> Optional[RoleType]: + """Get user's role in the project's organisations""" + user_org = ( + project.projectorganisation_set.filter( + organisation__organisationmembership__user=user + ) + .select_related("organisation__organisationmembership") + .first() + ) + + if not user_org: + return None + + return user_org.organisation.get_user_role(user) + + +def can_view_project(user: User, project: Project) -> bool: + """ + Check if user can view the project: + - ADMIN/MEMBER: Can always view if they're in the project's organisations + - GUEST: Can view if they have explicit VIEW or EDIT access + """ + role = get_user_role_in_project(user, project) + + if not role: + return False + + if role in [ROLE_ADMIN, ROLE_MEMBER]: + return True + elif role == ROLE_GUEST: + return GuestProjectAccess.objects.filter(user=user, project=project).exists() + + return False + + +def can_edit_project(user: User, project: Project) -> bool: + """ + Check if user can edit the project: + - ADMIN/MEMBER: Can always edit if they're in the project's organisations + - GUEST: Can only edit if they have explicit EDIT access + """ + role = get_user_role_in_project(user, project) + + if not role: + return False + + if role in [ROLE_ADMIN, ROLE_MEMBER]: + return True + elif role == ROLE_GUEST: + return GuestProjectAccess.objects.filter( + user=user, project=project, permission="EDIT" + ).exists() + + return False diff --git a/home/services.py b/home/services.py new file mode 100644 index 0000000..63075d2 --- /dev/null +++ b/home/services.py @@ -0,0 +1,34 @@ +from .models import GuestProjectAccess, Project, User + + +class ProjectAccessService: + """ + Service class for managing project access + """ + + @staticmethod + def grant_guest_access(project: Project, guest_user: User, granted_by: User, permission ="VIEW"): + if permission not in ["VIEW", "EDIT"]: + raise ValueError("Permission must be either VIEW or EDIT") + + if not guest_user.organisationmembership_set.filter( + organisation__projectorganisation__project=project, + role="GUEST" + ).exists(): + raise ValueError("User must be a guest in one of the project's organisations") + + return GuestProjectAccess.objects.update_or_create( + user=guest_user, + project=project, + defaults={ + 'granted_by': granted_by, + 'permission': permission + } + ) + + @staticmethod + def revoke_guest_access(project, guest_user): + return GuestProjectAccess.objects.filter( + user=guest_user, + project=project + ).delete() \ No newline at end of file diff --git a/home/views.py b/home/views.py index adfa25e..86b9780 100644 --- a/home/views.py +++ b/home/views.py @@ -10,7 +10,7 @@ from survey.models import Survey from .mixins import OrganisationRequiredMixin from .models import Organisation, Project, OrganisationMembership, ProjectOrganisation -from django.shortcuts import render +from django.shortcuts import get_object_or_404, render from django.views import View from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm from django.contrib.auth import login @@ -26,6 +26,9 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count +from .permissions import can_view_project, can_edit_project +from .services import ProjectAccessService + from django.core.exceptions import PermissionDenied from django.urls import reverse from django.shortcuts import get_object_or_404 @@ -134,7 +137,7 @@ def get_context_data(self, **kwargs): context["organisation"] = organisation context["can_edit"] = { - project.id: project.user_can_edit(self.request.user) + project.id: can_edit_project(self.request.user, project) for project in context["projects"] } context["can_create"] = OrganisationMembership.objects.filter( @@ -175,23 +178,34 @@ def form_valid(self, form): return redirect("myorganisation") - class ProjectView(LoginRequiredMixin, ListView): template_name = "projects/project.html" context_object_name = "surveys" paginate_by = 10 + def dispatch(self, request, *args, **kwargs): + # Check if user is allowed to access the project + try: + self.project = Project.objects.get(id=self.kwargs["project_id"]) + except Project.DoesNotExist: + messages.error(request, "Project not found.") + return redirect("myorganisation") + + if not can_view_project(request.user, self.project): + messages.error( + request, + f"You do not have permission to view the project {self.project.name}.", + ) + return redirect("myorganisation") + return super().dispatch(request, *args, **kwargs) + def get_queryset(self): - surveys = Survey.objects.filter(project_id=self.kwargs["project_id"]) - return surveys + return Survey.objects.filter(project_id=self.kwargs["project_id"]) def get_context_data(self, **kwargs): - # TODO: Check if user is allowed to access the project context = super().get_context_data(**kwargs) context["project"] = Project.objects.get(id=self.kwargs["project_id"]) - - # TODO: Check for role level for creating surveys - context["can_create"] = True + context["can_edit"] = can_edit_project(self.request.user, self.project) return context diff --git a/static/templates/base.html b/static/templates/base.html index b22f713..fc9d9c9 100644 --- a/static/templates/base.html +++ b/static/templates/base.html @@ -63,22 +63,14 @@ {% if messages %} {% for message in messages %} {% if message.tags == 'success' %} - + {% endif %} {% if message.tags == 'error' %} - + {% elif message.tags == 'warning' %} - + {% elif message.tags == 'info' %} - + {% endif %} {% endfor %} {% endif %} diff --git a/survey/migrations/0004_alter_surveyresponse_survey.py b/survey/migrations/0004_alter_surveyresponse_survey.py new file mode 100644 index 0000000..e78b039 --- /dev/null +++ b/survey/migrations/0004_alter_surveyresponse_survey.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.2 on 2025-01-15 19:01 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('survey', '0003_alter_survey_description_alter_survey_survey_config'), + ] + + operations = [ + migrations.AlterField( + model_name='surveyresponse', + name='survey', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='survey_response', to='survey.survey'), + ), + ] From e2695a5ce77f58c39bca204b0412d5114837e1af Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Wed, 15 Jan 2025 19:49:37 +0000 Subject: [PATCH 63/79] fix: select only organisation --- home/models.py | 2 -- home/permissions.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/home/models.py b/home/models.py index 5e24b8d..d0e5cdf 100644 --- a/home/models.py +++ b/home/models.py @@ -67,8 +67,6 @@ def get_user_role(self, user): class OrganisationMembership(models.Model): - ROLE_CHOICES = ROLES - user = models.ForeignKey(User, on_delete=models.CASCADE) organisation = models.ForeignKey(Organisation, on_delete=models.CASCADE) role = models.CharField(max_length=20, choices=ROLES, default=ROLE_GUEST) diff --git a/home/permissions.py b/home/permissions.py index bc1cd5f..fb698ac 100644 --- a/home/permissions.py +++ b/home/permissions.py @@ -9,7 +9,7 @@ def get_user_role_in_project(user: User, project: Project) -> Optional[RoleType] project.projectorganisation_set.filter( organisation__organisationmembership__user=user ) - .select_related("organisation__organisationmembership") + .select_related("organisation") .first() ) From b26b6098ab70a072d3bc15e38bf3aec11de5183b Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Wed, 15 Jan 2025 19:56:26 +0000 Subject: [PATCH 64/79] refactor: organisation services --- home/permissions.py | 15 +++++++-- home/services.py | 74 ++++++++++++++++++++++++++++++++++++--------- home/views.py | 61 +++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 45 deletions(-) diff --git a/home/permissions.py b/home/permissions.py index fb698ac..315d3a2 100644 --- a/home/permissions.py +++ b/home/permissions.py @@ -1,5 +1,6 @@ -from typing import Optional -from .models import GuestProjectAccess, Project, User +from typing import Optional, Dict +from django.db.models.query import QuerySet +from .models import GuestProjectAccess, OrganisationMembership, Project, User from .constants import ROLE_ADMIN, ROLE_MEMBER, ROLE_GUEST, RoleType @@ -57,3 +58,13 @@ def can_edit_project(user: User, project: Project) -> bool: ).exists() return False + + +def can_create_projects(user: User) -> bool: + """Check if user can create projects based on admin role""" + return OrganisationMembership.objects.filter(user=user, role=ROLE_ADMIN).exists() + + +def get_project_permissions(user: User, projects: QuerySet[Project]) -> Dict[int, bool]: + """Get edit permissions for multiple projects""" + return {project.id: can_edit_project(user, project) for project in projects} diff --git a/home/services.py b/home/services.py index 63075d2..ad111b5 100644 --- a/home/services.py +++ b/home/services.py @@ -1,34 +1,80 @@ -from .models import GuestProjectAccess, Project, User +from .models import ( + GuestProjectAccess, + Organisation, + OrganisationMembership, + Project, + User, +) +from typing import Dict, List, Set +from django.db.models.query import QuerySet +from django.db.models import Count + + +class OrganisationService: + @staticmethod + def get_user_organisation(user: User) -> Organisation: + """Get user's primary organisation""" + return user.organisation_set.first() + + @staticmethod + def get_user_organisation_ids(user: User) -> Set[int]: + """Get IDs of organisations user belongs to""" + return set( + OrganisationMembership.objects.filter(user=user).values_list( + "organisation_id", flat=True + ) + ) + + @staticmethod + def get_organisation_projects(organisation: Organisation) -> QuerySet[Project]: + """Get projects for an organisation with survey count""" + return Project.objects.filter( + projectorganisation__organisation=organisation + ).annotate(survey_count=Count("survey")) + + +class OrganisationAccessService: + + @staticmethod + def get_user_accessible_organisations( + projects: QuerySet[Project], user_org_ids: Set[int] + ) -> Dict[int, List[Organisation]]: + """Get organisations for each project that user is member of""" + return { + project.id: [ + org for org in project.organisations.all() if org.id in user_org_ids + ] + for project in projects + } class ProjectAccessService: """ Service class for managing project access """ - + @staticmethod - def grant_guest_access(project: Project, guest_user: User, granted_by: User, permission ="VIEW"): + def grant_guest_access( + project: Project, guest_user: User, granted_by: User, permission="VIEW" + ): if permission not in ["VIEW", "EDIT"]: raise ValueError("Permission must be either VIEW or EDIT") if not guest_user.organisationmembership_set.filter( - organisation__projectorganisation__project=project, - role="GUEST" + organisation__projectorganisation__project=project, role="GUEST" ).exists(): - raise ValueError("User must be a guest in one of the project's organisations") - + raise ValueError( + "User must be a guest in one of the project's organisations" + ) + return GuestProjectAccess.objects.update_or_create( user=guest_user, project=project, - defaults={ - 'granted_by': granted_by, - 'permission': permission - } + defaults={"granted_by": granted_by, "permission": permission}, ) @staticmethod def revoke_guest_access(project, guest_user): return GuestProjectAccess.objects.filter( - user=guest_user, - project=project - ).delete() \ No newline at end of file + user=guest_user, project=project + ).delete() diff --git a/home/views.py b/home/views.py index 86b9780..417473b 100644 --- a/home/views.py +++ b/home/views.py @@ -26,8 +26,17 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count -from .permissions import can_view_project, can_edit_project -from .services import ProjectAccessService +from .permissions import ( + can_create_projects, + can_view_project, + can_edit_project, + get_project_permissions, +) +from .services import ( + OrganisationAccessService, + OrganisationService, + ProjectAccessService, +) from django.core.exceptions import PermissionDenied from django.urls import reverse @@ -124,39 +133,33 @@ class MyOrganisationView(LoginRequiredMixin, OrganisationRequiredMixin, ListView context_object_name = "projects" paginate_by = 10 + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) + self.org_service = OrganisationService() + self.org_access_service = OrganisationAccessService() + self.organisation = self.org_service.get_user_organisation(request.user) + def get_queryset(self): - organisation = self.request.user.organisation_set.first() - projects = Project.objects.filter( - projectorganisation__organisation=organisation - ).annotate(survey_count=Count("survey")) - return projects + return self.org_service.get_organisation_projects(self.organisation) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - organisation = self.request.user.organisation_set.first() - context["organisation"] = organisation - - context["can_edit"] = { - project.id: can_edit_project(self.request.user, project) - for project in context["projects"] - } - context["can_create"] = OrganisationMembership.objects.filter( - user=self.request.user, role="ADMIN" - ).exists() - - user_orgs = set( - OrganisationMembership.objects.filter(user=self.request.user).values_list( - "organisation_id", flat=True - ) + projects = context["projects"] + user = self.request.user + + user_org_ids = self.org_service.get_user_organisation_ids(user) + + context.update( + { + "organisation": self.organisation, + "can_edit": get_project_permissions(user, projects), + "can_create": can_create_projects(user), + "project_orgs": self.org_access_service.get_user_accessible_organisations( + projects, user_org_ids + ), + } ) - context["project_orgs"] = { - project.id: [ - org for org in project.organisations.all() if org.id in user_orgs - ] - for project in context["projects"] - } - return context From 7046055a6e1da779bae923302912e05eaac61831 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:51:24 +0000 Subject: [PATCH 65/79] refactor: using services --- home/permissions.py | 15 +++++++++++++++ home/services.py | 22 +++++++++++++++++++++- home/views.py | 14 +++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/home/permissions.py b/home/permissions.py index 315d3a2..50de2d0 100644 --- a/home/permissions.py +++ b/home/permissions.py @@ -1,3 +1,18 @@ +""" +This module handles permission checking logic. + +Permissions files define who can do what in the system. They contain functions that: +- Check if users have specific permissions +- Validate access rights +- Determine role-based capabilities +- Handle authorization logic + +These functions should: +- Take a user and the object to check permissions against +- Return boolean or permission mapping results +- Be pure functions when possible (same input always gives same output) +- Not contain business logic (that belongs in services) +""" from typing import Optional, Dict from django.db.models.query import QuerySet from .models import GuestProjectAccess, OrganisationMembership, Project, User diff --git a/home/services.py b/home/services.py index ad111b5..ac88de8 100644 --- a/home/services.py +++ b/home/services.py @@ -1,3 +1,23 @@ +""" +This module contains the business logic and data operations. + +Services files are responsible for: +- Implementing business logic +- Handling data operations +- Coordinating between different parts of the system +- Providing an interface for views to access functionality + +Services should: +- Be organised by domain/model +- Handle complex operations +- Not contain permission checks (use permissions.py) +- Not contain presentation logic (use views) + +Example usage: + org_service = OrganisationService() + user_org = org_service.get_user_organisation(user) + projects = org_service.get_organisation_projects(user_org) +""" from .models import ( GuestProjectAccess, Organisation, @@ -50,7 +70,7 @@ def get_user_accessible_organisations( class ProjectAccessService: """ - Service class for managing project access + Service class for managing project access permissions for GUEST users """ @staticmethod diff --git a/home/views.py b/home/views.py index 417473b..d38ca23 100644 --- a/home/views.py +++ b/home/views.py @@ -217,12 +217,19 @@ class ProjectCreateView(LoginRequiredMixin, CreateView): model = Project fields = ["name"] template_name = "projects/create.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["organisation"] = Organisation.objects.get(id=self.kwargs["organisation_id"]) + + if not can_create_projects(self.request.user): + messages.error(self.request, f"You do not have the permission to create projects in this organisation.") + return redirect('myorganisation') def get_success_url(self): return self.object.get_absolute_url() def form_valid(self, form): - # TODO: Check user allowed to create project in the org result = super().form_valid(form) organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) # Link to the organisation @@ -249,8 +256,9 @@ def get_object(self, queryset=None): ) # Check if user has edit permissions - if not project.user_can_edit(self.request.user): - raise PermissionDenied("You don't have permission to edit this project.") + if not can_edit_project(self.request.user, project): + messages.error(self.request, "You don't have permission to edit this project.") + return redirect("myorganisation") return project From 41b298ddc55d421cae94b33e14956efa98dce0c1 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:49:19 +0000 Subject: [PATCH 66/79] refactor: remove guest user --- home/constants.py | 20 +++-- home/models.py | 22 ++---- home/permissions.py | 70 ++++++++--------- home/services.py | 186 +++++++++++++++++++++++++++++++++++--------- home/views.py | 121 ++++++++++++++++------------ 5 files changed, 276 insertions(+), 143 deletions(-) diff --git a/home/constants.py b/home/constants.py index 2cbeeb3..43a9d68 100644 --- a/home/constants.py +++ b/home/constants.py @@ -1,19 +1,23 @@ from typing import Literal ROLE_ADMIN = "ADMIN" -ROLE_MEMBER = "MEMBER" -ROLE_GUEST = "GUEST" +ROLE_PROJECT_MANAGER = "PROJECT_MANAGER" ROLES = [ (ROLE_ADMIN, "Admin"), - (ROLE_MEMBER, "Member"), - (ROLE_GUEST, "Guest"), + (ROLE_PROJECT_MANAGER, "Project Manager"), ] """ -ADMIN: Full control -MEMBER: Can view and edit projects -GUEST: Can view certain projects +ADMIN: Full control over organisation and all projects +PROJECT_MANAGER: Can manage specific projects with view or edit permissions """ +RoleType = Literal["ADMIN", "PROJECT_MANAGER"] -RoleType = Literal["ADMIN", "MEMBER", "GUEST"] \ No newline at end of file +PERMISSION_VIEW = "VIEW" +PERMISSION_EDIT = "EDIT" + +PERMISSION_CHOICES = [ + (PERMISSION_VIEW, "View Only"), + (PERMISSION_EDIT, "View and Edit"), +] \ No newline at end of file diff --git a/home/models.py b/home/models.py index d0e5cdf..0d8cebe 100644 --- a/home/models.py +++ b/home/models.py @@ -5,7 +5,7 @@ PermissionsMixin, ) from django.urls import reverse -from .constants import ROLES, ROLE_ADMIN, ROLE_MEMBER, ROLE_GUEST, RoleType +from .constants import ROLES, ROLE_PROJECT_MANAGER, PERMISSION_CHOICES, PERMISSION_VIEW class UserManager(BaseUserManager): @@ -69,27 +69,22 @@ def get_user_role(self, user): class OrganisationMembership(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) organisation = models.ForeignKey(Organisation, on_delete=models.CASCADE) - role = models.CharField(max_length=20, choices=ROLES, default=ROLE_GUEST) + role = models.CharField(max_length=20, choices=ROLES, default=ROLE_PROJECT_MANAGER) joined_at = models.DateTimeField(auto_now_add=True) class Meta: unique_together = ["user", "organisation"] - """ A user can only be a member of an organisation once """ class Project(models.Model): name = models.CharField(max_length=100) organisations = models.ManyToManyField(Organisation, through="ProjectOrganisation") - """ A project can be associated with multiple organisations/teams """ created_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True) created_on = models.DateTimeField(auto_now_add=True) def __str__(self): return self.name - def get_guest_users(self): - return GuestProjectAccess.objects.filter(project=self).select_related("user") - def get_absolute_url(self): return reverse("project", kwargs={"project_id": self.pk}) @@ -104,22 +99,19 @@ class Meta: unique_together = ["project", "organisation"] -class GuestProjectAccess(models.Model): - PERMISSION_CHOICES = [ - ("VIEW", "View Only"), - ("EDIT", "View and Edit"), - ] +class ProjectManagerPermission(models.Model): + """Defines the permission level for project managers within a project""" user = models.ForeignKey(User, on_delete=models.CASCADE) project = models.ForeignKey(Project, on_delete=models.CASCADE) permission = models.CharField( - max_length=10, choices=PERMISSION_CHOICES, default="VIEW" + max_length=10, choices=PERMISSION_CHOICES, default=PERMISSION_VIEW ) granted_by = models.ForeignKey( - User, on_delete=models.SET_NULL, null=True, related_name="granted_access" + User, on_delete=models.SET_NULL, null=True, related_name="granted_permissions" ) granted_at = models.DateTimeField(auto_now_add=True) class Meta: unique_together = ["user", "project"] - verbose_name_plural = "Guest project access" + verbose_name_plural = "Project manager permissions" diff --git a/home/permissions.py b/home/permissions.py index 50de2d0..61c0197 100644 --- a/home/permissions.py +++ b/home/permissions.py @@ -13,43 +13,44 @@ - Be pure functions when possible (same input always gives same output) - Not contain business logic (that belongs in services) """ + from typing import Optional, Dict from django.db.models.query import QuerySet -from .models import GuestProjectAccess, OrganisationMembership, Project, User -from .constants import ROLE_ADMIN, ROLE_MEMBER, ROLE_GUEST, RoleType - - -def get_user_role_in_project(user: User, project: Project) -> Optional[RoleType]: - """Get user's role in the project's organisations""" - user_org = ( - project.projectorganisation_set.filter( - organisation__organisationmembership__user=user - ) - .select_related("organisation") - .first() +from .models import Project, User +from .constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER +from .services import ProjectService, OrganisationService + + +def get_user_role_in_project(user: User, project: Project) -> Optional[str]: + """Get user's highest role across project's organisations""" + project_orgs = project.organisations.all() + for org in project_orgs: + role = org.get_user_role(user) + if role == ROLE_ADMIN: + return ROLE_ADMIN + # Return PROJECT_MANAGER if found in any org, otherwise None + return next( + ( + org.get_user_role(user) + for org in project_orgs + if org.get_user_role(user) == ROLE_PROJECT_MANAGER + ), + None, ) - if not user_org: - return None - - return user_org.organisation.get_user_role(user) - def can_view_project(user: User, project: Project) -> bool: """ Check if user can view the project: - - ADMIN/MEMBER: Can always view if they're in the project's organisations - - GUEST: Can view if they have explicit VIEW or EDIT access + - ADMIN: Can always view if they're in the project's organisations + - PROJECT_MANAGER: Can view if they have explicit permission """ role = get_user_role_in_project(user, project) - if not role: - return False - - if role in [ROLE_ADMIN, ROLE_MEMBER]: + if role == ROLE_ADMIN: return True - elif role == ROLE_GUEST: - return GuestProjectAccess.objects.filter(user=user, project=project).exists() + elif role == ROLE_PROJECT_MANAGER: + return ProjectService.get_user_permission(project, user) is not None return False @@ -57,27 +58,24 @@ def can_view_project(user: User, project: Project) -> bool: def can_edit_project(user: User, project: Project) -> bool: """ Check if user can edit the project: - - ADMIN/MEMBER: Can always edit if they're in the project's organisations - - GUEST: Can only edit if they have explicit EDIT access + - ADMIN: Can always edit if they're in the project's organisations + - PROJECT_MANAGER: Can only edit if they have explicit EDIT permission """ role = get_user_role_in_project(user, project) - if not role: - return False - - if role in [ROLE_ADMIN, ROLE_MEMBER]: + if role == ROLE_ADMIN: return True - elif role == ROLE_GUEST: - return GuestProjectAccess.objects.filter( - user=user, project=project, permission="EDIT" - ).exists() + elif role == ROLE_PROJECT_MANAGER: + permission = ProjectService.get_user_permission(project, user) + return permission and permission.permission == "EDIT" return False def can_create_projects(user: User) -> bool: """Check if user can create projects based on admin role""" - return OrganisationMembership.objects.filter(user=user, role=ROLE_ADMIN).exists() + org = OrganisationService.get_user_organisation(user) + return org and org.get_user_role(user) == ROLE_ADMIN def get_project_permissions(user: User, projects: QuerySet[Project]) -> Dict[int, bool]: diff --git a/home/services.py b/home/services.py index ac88de8..97ae902 100644 --- a/home/services.py +++ b/home/services.py @@ -18,21 +18,114 @@ user_org = org_service.get_user_organisation(user) projects = org_service.get_organisation_projects(user_org) """ + from .models import ( - GuestProjectAccess, + ProjectManagerPermission, Organisation, OrganisationMembership, Project, + ProjectOrganisation, User, ) -from typing import Dict, List, Set +from typing import Dict, List, Literal, Set, Optional, Union from django.db.models.query import QuerySet from django.db.models import Count +from .constants import ROLE_PROJECT_MANAGER, ROLE_ADMIN + + +class ProjectService: + """ + Service class for managing project operations and permissions + """ + + @staticmethod + def get_user_permission( + project: Project, user: User + ) -> Optional[ProjectManagerPermission]: + """Get user's permission level for a specific project""" + return ProjectManagerPermission.objects.filter( + user=user, project=project + ).first() + + @staticmethod + def grant_permission( + project: Project, + project_manager: User, + granted_by: User, + permission: Literal["VIEW", "EDIT"] = "VIEW", + ) -> tuple[ProjectManagerPermission, bool]: + """ + Grant project permission to a project manager + Returns tuple of (permission_object, created) + """ + if permission not in ["VIEW", "EDIT"]: + raise ValueError("Permission must be either VIEW or EDIT") + + # Verify user is a project manager in one of the project's organisations + if not project_manager.organisationmembership_set.filter( + organisation__projectorganisation__project=project, + role=ROLE_PROJECT_MANAGER, + ).exists(): + raise ValueError( + "User must be a project manager in one of the project's organisations" + ) + + return ProjectManagerPermission.objects.update_or_create( + user=project_manager, + project=project, + defaults={"granted_by": granted_by, "permission": permission}, + ) + + @staticmethod + def revoke_permission(project: Project, project_manager: User) -> tuple[int, dict]: + """ + Revoke all permissions for a project manager on a project + Returns tuple of (deleted_count, deleted_objects_by_type) + """ + return ProjectManagerPermission.objects.filter( + user=project_manager, project=project + ).delete() + + @staticmethod + def link_project_to_organisation( + project: Project, + organisation: Organisation, + user: User, + permission: Literal["VIEW", "EDIT"] = "EDIT", + ) -> ProjectOrganisation: + """ + Links a project to an organisation and handles project manager permissions + Also grants appropriate permissions if user is a project manager + """ + project_org = ProjectOrganisation.objects.create( + project=project, organisation=organisation, added_by=user + ) + + # Handle project manager permissions if applicable + user_role = organisation.get_user_role(user) + if user_role == ROLE_PROJECT_MANAGER: + ProjectService.grant_permission( + project=project, + project_manager=user, + granted_by=user, + permission=permission, + ) + + return project_org + + @staticmethod + def get_user_projects(user: User) -> QuerySet[Project]: + """Get all projects a user has access to""" + return Project.objects.filter(projectmanagerpermission__user=user).distinct() class OrganisationService: + """ + Service class for managing organisation operations and access + """ + @staticmethod - def get_user_organisation(user: User) -> Organisation: + def get_user_organisation(user: User) -> Optional[Organisation]: """Get user's primary organisation""" return user.organisation_set.first() @@ -45,21 +138,12 @@ def get_user_organisation_ids(user: User) -> Set[int]: ) ) - @staticmethod - def get_organisation_projects(organisation: Organisation) -> QuerySet[Project]: - """Get projects for an organisation with survey count""" - return Project.objects.filter( - projectorganisation__organisation=organisation - ).annotate(survey_count=Count("survey")) - - -class OrganisationAccessService: - @staticmethod def get_user_accessible_organisations( - projects: QuerySet[Project], user_org_ids: Set[int] + projects: QuerySet[Project], user: User ) -> Dict[int, List[Organisation]]: """Get organisations for each project that user is member of""" + user_org_ids = OrganisationService.get_user_organisation_ids(user) return { project.id: [ org for org in project.organisations.all() if org.id in user_org_ids @@ -67,34 +151,66 @@ def get_user_accessible_organisations( for project in projects } + @staticmethod + def get_organisation_projects( + organisation: Organisation, with_metrics: bool = True + ) -> QuerySet[Project]: + """ + Get projects for an organisation, optionally with metrics + Metrics include: survey count, manager count + """ + projects = Project.objects.filter( + projectorganisation__organisation=organisation + ) -class ProjectAccessService: - """ - Service class for managing project access permissions for GUEST users - """ + if with_metrics: + projects = projects.annotate( + survey_count=Count("survey"), + manager_count=Count("projectmanagerpermission", distinct=True), + ) - @staticmethod - def grant_guest_access( - project: Project, guest_user: User, granted_by: User, permission="VIEW" - ): - if permission not in ["VIEW", "EDIT"]: - raise ValueError("Permission must be either VIEW or EDIT") + return projects - if not guest_user.organisationmembership_set.filter( - organisation__projectorganisation__project=project, role="GUEST" - ).exists(): + @staticmethod + def add_user_to_organisation( + user: User, + organisation: Organisation, + role: Literal["ADMIN", "PROJECT_MANAGER"], + added_by: User, + ) -> OrganisationMembership: + """Add a user to an organisation with specified role""" + if role not in [ROLE_ADMIN, ROLE_PROJECT_MANAGER]: raise ValueError( - "User must be a guest in one of the project's organisations" + f"Role must be either {ROLE_ADMIN} or {ROLE_PROJECT_MANAGER}" ) - return GuestProjectAccess.objects.update_or_create( - user=guest_user, - project=project, - defaults={"granted_by": granted_by, "permission": permission}, + return OrganisationMembership.objects.create( + user=user, organisation=organisation, role=role ) @staticmethod - def revoke_guest_access(project, guest_user): - return GuestProjectAccess.objects.filter( - user=guest_user, project=project + def remove_user_from_organisation( + user: User, organisation: Organisation + ) -> tuple[int, dict]: + """ + Remove user from organisation and cleanup related permissions + Returns tuple of (deleted_count, deleted_objects_by_type) + """ + # First revoke all project permissions in this org + ProjectManagerPermission.objects.filter( + user=user, project__organisations=organisation + ).delete() + + # Then remove org membership + return OrganisationMembership.objects.filter( + user=user, organisation=organisation ).delete() + + @staticmethod + def get_organisation_members( + organisation: Organisation, + ) -> QuerySet[OrganisationMembership]: + """Get all members of an organisation with their roles""" + return OrganisationMembership.objects.filter( + organisation=organisation + ).select_related("user") diff --git a/home/views.py b/home/views.py index d38ca23..e2aa482 100644 --- a/home/views.py +++ b/home/views.py @@ -33,11 +33,12 @@ get_project_permissions, ) from .services import ( - OrganisationAccessService, OrganisationService, - ProjectAccessService, + ProjectService ) +from .constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER + from django.core.exceptions import PermissionDenied from django.urls import reverse from django.shortcuts import get_object_or_404 @@ -135,27 +136,27 @@ class MyOrganisationView(LoginRequiredMixin, OrganisationRequiredMixin, ListView def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) - self.org_service = OrganisationService() - self.org_access_service = OrganisationAccessService() - self.organisation = self.org_service.get_user_organisation(request.user) + self.organisation = OrganisationService.get_user_organisation(request.user) def get_queryset(self): - return self.org_service.get_organisation_projects(self.organisation) + return OrganisationService.get_organisation_projects(self.organisation) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - projects = context["projects"] user = self.request.user + projects = context["projects"] - user_org_ids = self.org_service.get_user_organisation_ids(user) + user_role = self.organisation.get_user_role(user) context.update( { "organisation": self.organisation, "can_edit": get_project_permissions(user, projects), "can_create": can_create_projects(user), - "project_orgs": self.org_access_service.get_user_accessible_organisations( - projects, user_org_ids + "is_admin": user_role == ROLE_ADMIN, + "is_project_manager": user_role == ROLE_PROJECT_MANAGER, + "project_orgs": OrganisationService.get_user_accessible_organisations( + projects, user ), } ) @@ -173,11 +174,12 @@ def get_success_url(self): def form_valid(self, form): super().form_valid(form) - # Add user that creates the org as admin - OrganisationMembership.objects.create( - organisation=self.object, user=self.request.user, role="ADMIN" + OrganisationService.add_user_to_organisation( + user=self.request.user, + organisation=self.object, + role=ROLE_ADMIN, + added_by=self.request.user, ) - return redirect("myorganisation") @@ -187,7 +189,6 @@ class ProjectView(LoginRequiredMixin, ListView): paginate_by = 10 def dispatch(self, request, *args, **kwargs): - # Check if user is allowed to access the project try: self.project = Project.objects.get(id=self.kwargs["project_id"]) except Project.DoesNotExist: @@ -207,8 +208,18 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["project"] = Project.objects.get(id=self.kwargs["project_id"]) - context["can_edit"] = can_edit_project(self.request.user, self.project) + user = self.request.user + project = self.project + + permission = ProjectService.get_user_permission(project, user) + + context.update( + { + "project": project, + "can_edit": can_edit_project(user, project), + "permission_level": permission.permission if permission else None, + } + ) return context @@ -217,25 +228,36 @@ class ProjectCreateView(LoginRequiredMixin, CreateView): model = Project fields = ["name"] template_name = "projects/create.html" - + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["organisation"] = Organisation.objects.get(id=self.kwargs["organisation_id"]) + organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) + context["organisation"] = organisation if not can_create_projects(self.request.user): - messages.error(self.request, f"You do not have the permission to create projects in this organisation.") - return redirect('myorganisation') + messages.error( + self.request, + "You don't have permissions to create projects in this organisation.", + ) + return redirect("myorganisation") + + return context def get_success_url(self): return self.object.get_absolute_url() def form_valid(self, form): + form.instance.created_by = self.request.user result = super().form_valid(form) + organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) - # Link to the organisation - ProjectOrganisation.objects.create( - project=self.object, organisation=organisation, added_by=self.request.user + ProjectService.link_project_to_organisation( + project=self.object, + organisation=organisation, + user=self.request.user, + permission="EDIT", # Project creators get edit permission by default ) + return result @@ -246,44 +268,43 @@ class ProjectEditView(LoginRequiredMixin, UpdateView): context_object_name = "project" def get_object(self, queryset=None): - # Get the project with related organizations project = get_object_or_404( Project.objects.prefetch_related( - "organisations", - "organisations__organisationmembership_set" + "organisations", "organisations__organisationmembership_set" ), - id=self.kwargs["project_id"] + id=self.kwargs["project_id"], ) - # Check if user has edit permissions if not can_edit_project(self.request.user, project): - messages.error(self.request, "You don't have permission to edit this project.") + messages.error( + self.request, "You don't have permission to edit this project." + ) return redirect("myorganisation") return project def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) + user = self.request.user - # Get user's org - user_orgs = set( - OrganisationMembership.objects.filter( - user=self.request.user - ).values_list("organisation_id", flat=True) - ) + # Get all organisations user has access to + project_orgs = OrganisationService.get_user_accessible_organisations( + [self.object], user + ).get(self.object.id, []) - # Get the org that the user is a member of and are linked to the project - context["project_orgs"] = [ - org for org in self.object.organisations.all() - if org.id in user_orgs - ] - - # Check if user can manage org for this project - context["can_manage_orgs"] = any( - membership.role == "ADMIN" - for org in context["project_orgs"] - for membership in org.organisationmembership_set.all() - if membership.user == self.request.user + # Get user's roles across organisations + user_roles = {org.id: org.get_user_role(user) for org in project_orgs} + + context.update( + { + "project_orgs": project_orgs, + "can_manage_orgs": any( + role == ROLE_ADMIN for role in user_roles.values() + ), + "is_project_manager": any( + role == ROLE_PROJECT_MANAGER for role in user_roles.values() + ), + } ) return context @@ -292,6 +313,8 @@ def get_success_url(self): return reverse("myorganisation") def form_valid(self, form): - # Perform the update response = super().form_valid(form) + messages.success( + self.request, f"Project {self.object.name} has been updated successfully." + ) return response From 58c670a264fd3d867599fac26d69769d907e121b Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 12:56:30 +0000 Subject: [PATCH 67/79] refactor: test data --- data/003_memberships.json | 6 +-- data/006_project_managers.json | 24 ++++++++++++ data/{006_surveys.json => 007_surveys.json} | 0 ...sponses.json => 008_survey_responses.json} | 0 data/README.md | 2 +- ...er_organisationmembership_role_and_more.py | 38 +++++++++++++++++++ 6 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 data/006_project_managers.json rename data/{006_surveys.json => 007_surveys.json} (100%) rename data/{007_survey_responses.json => 008_survey_responses.json} (100%) create mode 100644 home/migrations/0004_alter_organisationmembership_role_and_more.py diff --git a/data/003_memberships.json b/data/003_memberships.json index eaa69cb..4217e5c 100644 --- a/data/003_memberships.json +++ b/data/003_memberships.json @@ -15,7 +15,7 @@ "fields": { "user": 2, "organisation": 1, - "role": "MEMBER", + "role": "PROJECT_MANAGER", "joined_at": "2024-01-15T00:00:00Z" } }, @@ -24,8 +24,8 @@ "pk": 3, "fields": { "user": 3, - "organisation": 1, - "role": "GUEST", + "organisation": 2, + "role": "PROJECT_MANAGER", "joined_at": "2024-01-15T00:00:00Z" } } diff --git a/data/006_project_managers.json b/data/006_project_managers.json new file mode 100644 index 0000000..0923404 --- /dev/null +++ b/data/006_project_managers.json @@ -0,0 +1,24 @@ +[ + { + "model": "home.projectmanagerpermission", + "pk": 1, + "fields": { + "user": 2, + "project": 1, + "permission": "EDIT", + "granted_by": 1, + "granted_at": "2024-01-15T00:00:00Z" + } + }, + { + "model": "home.projectmanagerpermission", + "pk": 2, + "fields": { + "user": 3, + "project": 1, + "permission": "VIEW", + "granted_by": 1, + "granted_at": "2024-01-15T00:00:00Z" + } + } +] diff --git a/data/006_surveys.json b/data/007_surveys.json similarity index 100% rename from data/006_surveys.json rename to data/007_surveys.json diff --git a/data/007_survey_responses.json b/data/008_survey_responses.json similarity index 100% rename from data/007_survey_responses.json rename to data/008_survey_responses.json diff --git a/data/README.md b/data/README.md index ea48af6..bf7d4f4 100644 --- a/data/README.md +++ b/data/README.md @@ -3,7 +3,7 @@ Run: ```bash -python manage.py loaddata ./data/001_users.json ./data/002_organisations.json ./data/003_memberships.json ./data/004_projects.json ./data/005_project_organisations.json ./data/006_surveys.json ./data/007_survey_responses.json +python manage.py loaddata ./data/001_users.json ./data/002_organisations.json ./data/003_memberships.json ./data/004_projects.json ./data/005_project_organisations.json ./data/006_project_managers.json ./data/007_surveys.json ./data/008_survey_responses.json ``` The password for users is their respective role name in lowercases. diff --git a/home/migrations/0004_alter_organisationmembership_role_and_more.py b/home/migrations/0004_alter_organisationmembership_role_and_more.py new file mode 100644 index 0000000..3188412 --- /dev/null +++ b/home/migrations/0004_alter_organisationmembership_role_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 5.1.2 on 2025-01-16 12:50 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('home', '0003_alter_organisationmembership_role_guestprojectaccess'), + ] + + operations = [ + migrations.AlterField( + model_name='organisationmembership', + name='role', + field=models.CharField(choices=[('ADMIN', 'Admin'), ('PROJECT_MANAGER', 'Project Manager')], default='PROJECT_MANAGER', max_length=20), + ), + migrations.CreateModel( + name='ProjectManagerPermission', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('permission', models.CharField(choices=[('VIEW', 'View Only'), ('EDIT', 'View and Edit')], default='VIEW', max_length=10)), + ('granted_at', models.DateTimeField(auto_now_add=True)), + ('granted_by', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='granted_permissions', to=settings.AUTH_USER_MODEL)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='home.project')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name_plural': 'Project manager permissions', + 'unique_together': {('user', 'project')}, + }, + ), + migrations.DeleteModel( + name='GuestProjectAccess', + ), + ] From 00677894a0ea5c0ca5ff1298f2c2429ac3f8f87e Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:03:15 +0000 Subject: [PATCH 68/79] chore: permissions --- home/services.py | 1 + home/views.py | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/home/services.py b/home/services.py index 97ae902..80de1e1 100644 --- a/home/services.py +++ b/home/services.py @@ -43,6 +43,7 @@ def get_user_permission( project: Project, user: User ) -> Optional[ProjectManagerPermission]: """Get user's permission level for a specific project""" + return ProjectManagerPermission.objects.filter( user=user, project=project ).first() diff --git a/home/views.py b/home/views.py index e2aa482..ceade6d 100644 --- a/home/views.py +++ b/home/views.py @@ -211,13 +211,10 @@ def get_context_data(self, **kwargs): user = self.request.user project = self.project - permission = ProjectService.get_user_permission(project, user) - context.update( { "project": project, - "can_edit": can_edit_project(user, project), - "permission_level": permission.permission if permission else None, + "can_create": can_edit_project(user, project), } ) From dce72e26f53a2a5b90ea4dcd6d70908155ff0924 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 13:52:21 +0000 Subject: [PATCH 69/79] refactor: merge service and permission --- home/permissions.py | 83 ------------- home/services.py | 217 ---------------------------------- home/services/__init__.py | 15 +++ home/services/base.py | 60 ++++++++++ home/services/organisation.py | 156 ++++++++++++++++++++++++ home/services/project.py | 162 +++++++++++++++++++++++++ home/views.py | 147 +++++++++++------------ 7 files changed, 463 insertions(+), 377 deletions(-) delete mode 100644 home/permissions.py delete mode 100644 home/services.py create mode 100644 home/services/__init__.py create mode 100644 home/services/base.py create mode 100644 home/services/organisation.py create mode 100644 home/services/project.py diff --git a/home/permissions.py b/home/permissions.py deleted file mode 100644 index 61c0197..0000000 --- a/home/permissions.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -This module handles permission checking logic. - -Permissions files define who can do what in the system. They contain functions that: -- Check if users have specific permissions -- Validate access rights -- Determine role-based capabilities -- Handle authorization logic - -These functions should: -- Take a user and the object to check permissions against -- Return boolean or permission mapping results -- Be pure functions when possible (same input always gives same output) -- Not contain business logic (that belongs in services) -""" - -from typing import Optional, Dict -from django.db.models.query import QuerySet -from .models import Project, User -from .constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER -from .services import ProjectService, OrganisationService - - -def get_user_role_in_project(user: User, project: Project) -> Optional[str]: - """Get user's highest role across project's organisations""" - project_orgs = project.organisations.all() - for org in project_orgs: - role = org.get_user_role(user) - if role == ROLE_ADMIN: - return ROLE_ADMIN - # Return PROJECT_MANAGER if found in any org, otherwise None - return next( - ( - org.get_user_role(user) - for org in project_orgs - if org.get_user_role(user) == ROLE_PROJECT_MANAGER - ), - None, - ) - - -def can_view_project(user: User, project: Project) -> bool: - """ - Check if user can view the project: - - ADMIN: Can always view if they're in the project's organisations - - PROJECT_MANAGER: Can view if they have explicit permission - """ - role = get_user_role_in_project(user, project) - - if role == ROLE_ADMIN: - return True - elif role == ROLE_PROJECT_MANAGER: - return ProjectService.get_user_permission(project, user) is not None - - return False - - -def can_edit_project(user: User, project: Project) -> bool: - """ - Check if user can edit the project: - - ADMIN: Can always edit if they're in the project's organisations - - PROJECT_MANAGER: Can only edit if they have explicit EDIT permission - """ - role = get_user_role_in_project(user, project) - - if role == ROLE_ADMIN: - return True - elif role == ROLE_PROJECT_MANAGER: - permission = ProjectService.get_user_permission(project, user) - return permission and permission.permission == "EDIT" - - return False - - -def can_create_projects(user: User) -> bool: - """Check if user can create projects based on admin role""" - org = OrganisationService.get_user_organisation(user) - return org and org.get_user_role(user) == ROLE_ADMIN - - -def get_project_permissions(user: User, projects: QuerySet[Project]) -> Dict[int, bool]: - """Get edit permissions for multiple projects""" - return {project.id: can_edit_project(user, project) for project in projects} diff --git a/home/services.py b/home/services.py deleted file mode 100644 index 80de1e1..0000000 --- a/home/services.py +++ /dev/null @@ -1,217 +0,0 @@ -""" -This module contains the business logic and data operations. - -Services files are responsible for: -- Implementing business logic -- Handling data operations -- Coordinating between different parts of the system -- Providing an interface for views to access functionality - -Services should: -- Be organised by domain/model -- Handle complex operations -- Not contain permission checks (use permissions.py) -- Not contain presentation logic (use views) - -Example usage: - org_service = OrganisationService() - user_org = org_service.get_user_organisation(user) - projects = org_service.get_organisation_projects(user_org) -""" - -from .models import ( - ProjectManagerPermission, - Organisation, - OrganisationMembership, - Project, - ProjectOrganisation, - User, -) -from typing import Dict, List, Literal, Set, Optional, Union -from django.db.models.query import QuerySet -from django.db.models import Count -from .constants import ROLE_PROJECT_MANAGER, ROLE_ADMIN - - -class ProjectService: - """ - Service class for managing project operations and permissions - """ - - @staticmethod - def get_user_permission( - project: Project, user: User - ) -> Optional[ProjectManagerPermission]: - """Get user's permission level for a specific project""" - - return ProjectManagerPermission.objects.filter( - user=user, project=project - ).first() - - @staticmethod - def grant_permission( - project: Project, - project_manager: User, - granted_by: User, - permission: Literal["VIEW", "EDIT"] = "VIEW", - ) -> tuple[ProjectManagerPermission, bool]: - """ - Grant project permission to a project manager - Returns tuple of (permission_object, created) - """ - if permission not in ["VIEW", "EDIT"]: - raise ValueError("Permission must be either VIEW or EDIT") - - # Verify user is a project manager in one of the project's organisations - if not project_manager.organisationmembership_set.filter( - organisation__projectorganisation__project=project, - role=ROLE_PROJECT_MANAGER, - ).exists(): - raise ValueError( - "User must be a project manager in one of the project's organisations" - ) - - return ProjectManagerPermission.objects.update_or_create( - user=project_manager, - project=project, - defaults={"granted_by": granted_by, "permission": permission}, - ) - - @staticmethod - def revoke_permission(project: Project, project_manager: User) -> tuple[int, dict]: - """ - Revoke all permissions for a project manager on a project - Returns tuple of (deleted_count, deleted_objects_by_type) - """ - return ProjectManagerPermission.objects.filter( - user=project_manager, project=project - ).delete() - - @staticmethod - def link_project_to_organisation( - project: Project, - organisation: Organisation, - user: User, - permission: Literal["VIEW", "EDIT"] = "EDIT", - ) -> ProjectOrganisation: - """ - Links a project to an organisation and handles project manager permissions - Also grants appropriate permissions if user is a project manager - """ - project_org = ProjectOrganisation.objects.create( - project=project, organisation=organisation, added_by=user - ) - - # Handle project manager permissions if applicable - user_role = organisation.get_user_role(user) - if user_role == ROLE_PROJECT_MANAGER: - ProjectService.grant_permission( - project=project, - project_manager=user, - granted_by=user, - permission=permission, - ) - - return project_org - - @staticmethod - def get_user_projects(user: User) -> QuerySet[Project]: - """Get all projects a user has access to""" - return Project.objects.filter(projectmanagerpermission__user=user).distinct() - - -class OrganisationService: - """ - Service class for managing organisation operations and access - """ - - @staticmethod - def get_user_organisation(user: User) -> Optional[Organisation]: - """Get user's primary organisation""" - return user.organisation_set.first() - - @staticmethod - def get_user_organisation_ids(user: User) -> Set[int]: - """Get IDs of organisations user belongs to""" - return set( - OrganisationMembership.objects.filter(user=user).values_list( - "organisation_id", flat=True - ) - ) - - @staticmethod - def get_user_accessible_organisations( - projects: QuerySet[Project], user: User - ) -> Dict[int, List[Organisation]]: - """Get organisations for each project that user is member of""" - user_org_ids = OrganisationService.get_user_organisation_ids(user) - return { - project.id: [ - org for org in project.organisations.all() if org.id in user_org_ids - ] - for project in projects - } - - @staticmethod - def get_organisation_projects( - organisation: Organisation, with_metrics: bool = True - ) -> QuerySet[Project]: - """ - Get projects for an organisation, optionally with metrics - Metrics include: survey count, manager count - """ - projects = Project.objects.filter( - projectorganisation__organisation=organisation - ) - - if with_metrics: - projects = projects.annotate( - survey_count=Count("survey"), - manager_count=Count("projectmanagerpermission", distinct=True), - ) - - return projects - - @staticmethod - def add_user_to_organisation( - user: User, - organisation: Organisation, - role: Literal["ADMIN", "PROJECT_MANAGER"], - added_by: User, - ) -> OrganisationMembership: - """Add a user to an organisation with specified role""" - if role not in [ROLE_ADMIN, ROLE_PROJECT_MANAGER]: - raise ValueError( - f"Role must be either {ROLE_ADMIN} or {ROLE_PROJECT_MANAGER}" - ) - - return OrganisationMembership.objects.create( - user=user, organisation=organisation, role=role - ) - - @staticmethod - def remove_user_from_organisation( - user: User, organisation: Organisation - ) -> tuple[int, dict]: - """ - Remove user from organisation and cleanup related permissions - Returns tuple of (deleted_count, deleted_objects_by_type) - """ - # First revoke all project permissions in this org - ProjectManagerPermission.objects.filter( - user=user, project__organisations=organisation - ).delete() - - # Then remove org membership - return OrganisationMembership.objects.filter( - user=user, organisation=organisation - ).delete() - - @staticmethod - def get_organisation_members( - organisation: Organisation, - ) -> QuerySet[OrganisationMembership]: - """Get all members of an organisation with their roles""" - return OrganisationMembership.objects.filter( - organisation=organisation - ).select_related("user") diff --git a/home/services/__init__.py b/home/services/__init__.py new file mode 100644 index 0000000..2d7e82e --- /dev/null +++ b/home/services/__init__.py @@ -0,0 +1,15 @@ +from .base import BaseService +from .project import ProjectService +from .organisation import OrganisationService + +# Create instances for use in views +project_service = ProjectService() +organisation_service = OrganisationService() + +__all__ = [ + 'BaseService', + 'ProjectService', + 'OrganisationService', + 'project_service', + 'organisation_service' +] \ No newline at end of file diff --git a/home/services/base.py b/home/services/base.py new file mode 100644 index 0000000..62776af --- /dev/null +++ b/home/services/base.py @@ -0,0 +1,60 @@ +""" +Base service and permission decorators +""" + +from functools import wraps +from typing import TypeVar, Callable, Any +from django.core.exceptions import PermissionDenied + +T = TypeVar("T") + + +def requires_permission(permission_type: str, obj_param: str = "instance"): + """ + Permission decorator for service methods. + + Args: + permission_type: Type of permission to check (view/edit/delete) + obj_param: Name of the parameter that contains the object to check permissions against + """ + + def decorator(func: Callable) -> Callable: + @wraps(func) + def wrapper(service: Any, user: Any, *args, **kwargs) -> Any: + # Get the object to check permissions against + obj = kwargs.get(obj_param) or (args[0] if args else None) + if not obj: + raise ValueError(f"Could not find object parameter: {obj_param}") + + # Get the permission check method + check_method = getattr(service, f"can_{permission_type}") + if not check_method: + raise ValueError(f"Service does not implement: can_{permission_type}") + + # Check permission + if not check_method(user, obj): + raise PermissionDenied( + f"User does not have {permission_type} permission for {obj}" + ) + + return func(service, user, *args, **kwargs) + + return wrapper + + return decorator + + +class BasePermissionService: + """Base service class with permission checks""" + + def can_view(self, user: Any, instance: Any) -> bool: + raise NotImplementedError + + def can_edit(self, user: Any, instance: Any) -> bool: + raise NotImplementedError + + def can_delete(self, user: Any, instance: Any) -> bool: + raise NotImplementedError + + def can_create(self, user: Any) -> bool: + raise NotImplementedError diff --git a/home/services/organisation.py b/home/services/organisation.py new file mode 100644 index 0000000..a7b3504 --- /dev/null +++ b/home/services/organisation.py @@ -0,0 +1,156 @@ +""" +Organisation service with integrated permissions +""" + +from typing import Optional, Dict, List, Set, Literal +from django.db.models.query import QuerySet +from django.db.models import Count +from django.core.exceptions import PermissionDenied +from .base import BasePermissionService, requires_permission +from ..models import ( + Organisation, + User, + OrganisationMembership, + Project, + ProjectManagerPermission, +) +from ..constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER + + +class OrganisationService(BasePermissionService): + """Service for managing organisations with integrated permissions""" + + def get_user_role(self, user: User, organisation: Organisation) -> Optional[str]: + membership = organisation.organisationmembership_set.filter(user=user).first() + return membership.role if membership else None + + def can_view(self, user: User, organisation: Organisation) -> bool: + role = self.get_user_role(user, organisation) + return role in [ROLE_ADMIN, ROLE_PROJECT_MANAGER] + + def can_edit(self, user: User, organisation: Organisation) -> bool: + role = self.get_user_role(user, organisation) + return role == ROLE_ADMIN + + def can_create(self, user: User) -> bool: + return user.is_superuser + + def can_delete(self, user: User, organisation: Organisation) -> bool: + return user.is_superuser + + def can_manage_members(self, user: User, organisation: Organisation) -> bool: + role = self.get_user_role(user, organisation) + return role == ROLE_ADMIN + + @requires_permission("view") + def get_organisation(self, user: User, organisation: Organisation) -> Organisation: + """Get organisation if user has permission""" + return organisation + + def get_user_organisation(self, user: User) -> Optional[Organisation]: + """Get user's primary organisation""" + return user.organisation_set.first() + + def get_user_organisation_ids(self, user: User) -> Set[int]: + """Get IDs of organisations user belongs to""" + return set( + OrganisationMembership.objects.filter(user=user).values_list( + "organisation_id", flat=True + ) + ) + + def get_user_accessible_organisations( + self, projects: QuerySet[Project], user: User + ) -> Dict[int, List[Organisation]]: + """Get organisations for each project that user is member of""" + user_org_ids = self.get_user_organisation_ids(user) + return { + project.id: [ + org for org in project.organisations.all() if org.id in user_org_ids + ] + for project in projects + } + + @requires_permission("edit") + def update_organisation( + self, user: User, organisation: Organisation, data: Dict + ) -> Organisation: + """Update organization with provided data""" + for key, value in data.items(): + setattr(organisation, key, value) + organisation.save() + return organisation + + def create_organisation( + self, user: User, name: str, description: str = "" + ) -> Organisation: + """Create a new organization""" + if not self.can_create(user): + raise PermissionDenied("User cannot create organisations") + + org = Organisation.objects.create(name=name, description=description) + self.add_user_to_organisation( + user=user, organisation=org, role=ROLE_ADMIN, added_by=user + ) + return org + + @requires_permission("edit") + def add_user_to_organisation( + self, + user: User, + organisation: Organisation, + new_user: User, + role: Literal["ADMIN", "PROJECT_MANAGER"], + added_by: User, + ) -> OrganisationMembership: + """Add a user to an organisation with specified role""" + if role not in [ROLE_ADMIN, ROLE_PROJECT_MANAGER]: + raise ValueError( + f"Role must be either {ROLE_ADMIN} or {ROLE_PROJECT_MANAGER}" + ) + + return OrganisationMembership.objects.create( + user=new_user, organisation=organisation, role=role, added_by=added_by + ) + + @requires_permission("edit") + def remove_user_from_organisation( + self, user: User, organisation: Organisation, removed_user: User + ) -> None: + """Remove user from organisation and cleanup permissions""" + # First revoke all project permissions + ProjectManagerPermission.objects.filter( + user=removed_user, project__organisations=organisation + ).delete() + + # Then remove org membership + OrganisationMembership.objects.filter( + user=removed_user, organisation=organisation + ).delete() + + def get_organisation_projects( + self, organisation: Organisation, with_metrics: bool = True + ) -> QuerySet[Project]: + """Get projects for an organisation with optional metrics""" + + projects = Project.objects.filter( + projectorganisation__organisation=organisation + ) + + if with_metrics: + projects = projects.annotate( + survey_count=Count("survey"), + manager_count=Count("projectmanagerpermission", distinct=True), + ) + + return projects + + @requires_permission("view") + def get_organisation_members( + self, user: User, organisation: Organisation + ) -> QuerySet[OrganisationMembership]: + """Get all members of an organisation with their roles""" + + return OrganisationMembership.objects.filter( + organisation=organisation + ).select_related("user") diff --git a/home/services/project.py b/home/services/project.py new file mode 100644 index 0000000..9236c81 --- /dev/null +++ b/home/services/project.py @@ -0,0 +1,162 @@ +""" +Project service with integrated permissions +""" + +from typing import Optional, Dict, Tuple, Literal +from django.db.models.query import QuerySet +from .base import BasePermissionService, requires_permission +from ..models import ( + Project, + User, + Organisation, + ProjectManagerPermission, + ProjectOrganisation, +) +from ..constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER +from django.core.exceptions import PermissionDenied + + +class ProjectService(BasePermissionService): + """Service for managing projects with integrated permissions""" + + def get_user_role(self, user: User, project: Project) -> Optional[str]: + """Get user's highest role across project's organisations""" + project_orgs = project.organisations.all() + + # Check for admin role first + for org in project_orgs: + role = org.get_user_role(user) + if role == ROLE_ADMIN: + return ROLE_ADMIN + + # Then check for project manager role + return next( + ( + org.get_user_role(user) + for org in project_orgs + if org.get_user_role(user) == ROLE_PROJECT_MANAGER + ), + None, + ) + + def get_user_permission( + self, user: User, project: Project + ) -> Optional[ProjectManagerPermission]: + """Get user's permission level for a project""" + return ProjectManagerPermission.objects.filter( + user=user, project=project + ).first() + + def can_view(self, user: User, project: Project) -> bool: + role = self.get_user_role(user, project) + + if role == ROLE_ADMIN: + return True + elif role == ROLE_PROJECT_MANAGER: + return self.get_user_permission(project, user) is not None + + return False + + def can_edit(self, user: User, project: Project) -> bool: + role = self.get_user_role(user, project) + + if role == ROLE_ADMIN: + return True + elif role == ROLE_PROJECT_MANAGER: + permission = self.get_user_permission(project, user) + return permission and permission.permission == "EDIT" + + return False + + def can_create(self, user: User) -> bool: + org = user.organisation_set.first() + return org and org.get_user_role(user) == ROLE_ADMIN + + def can_delete(self, user: User, project: Project) -> bool: + role = self.get_user_role(user, project) + return role == ROLE_ADMIN + + @requires_permission("edit") + def update_project(self, user: User, project: Project, data: Dict) -> Project: + """Update project with provided data""" + for key, value in data.items(): + setattr(project, key, value) + + project.save() + return project + + @requires_permission("view") + def get_project(self, user: User, project: Project) -> Project: + """Get project if user has permission""" + return project + + def create_project( + self, user: User, name: str, organisation: Organisation + ) -> Project: + """Create a new project""" + if not self.can_create(user): + raise PermissionDenied("User cannot create projects") + + project = Project.objects.create(name=name, created_by=user) + self.link_project_to_organisation( + user=user, project=project, organisation=organisation, permission="EDIT" + ) + return project + + @requires_permission("edit", "project") + def grant_permission( + self, + user: User, + project: Project, + project_manager: User, + permission: Literal["VIEW", "EDIT"] = "VIEW", + ) -> Tuple[ProjectManagerPermission, bool]: + """Grant project permission to a project manager""" + + if permission not in ["VIEW", "EDIT"]: + raise ValueError("Permission must be either VIEW or EDIT") + + if not project_manager.organisationmembership_set.filter( + organisation__projectorganisation__project=project, + role=ROLE_PROJECT_MANAGER, + ).exists(): + raise ValueError("User must be a project manager") + + return ProjectManagerPermission.objects.update_or_create( + user=project_manager, + project=project, + defaults={"granted_by": user, "permission": permission}, + ) + + @requires_permission("edit", "project") + def revoke_permission( + self, user: User, project: Project, project_manager: User + ) -> None: + """Revoke permissions for a project manager""" + ProjectManagerPermission.objects.filter( + user=project_manager, project=project + ).delete() + + def link_project_to_organisation( + self, + user: User, + project: Project, + organisation: Organisation, + permission: Literal["VIEW", "EDIT"] = "EDIT", + ) -> ProjectOrganisation: + """Link project to organisation and handle permissions""" + project_org = ProjectOrganisation.objects.create( + project=project, organisation=organisation, added_by=user + ) + + user_role = organisation.get_user_role(user) + if user_role == ROLE_PROJECT_MANAGER: + self.grant_permission( + user=user, project=project, project_manager=user, permission=permission + ) + + return project_org + + def get_user_projects(self, user: User) -> QuerySet[Project]: + """Get all projects a user has access to""" + return Project.objects.filter(projectmanagerpermission__user=user).distinct() diff --git a/home/views.py b/home/views.py index ceade6d..26c51ff 100644 --- a/home/views.py +++ b/home/views.py @@ -1,48 +1,29 @@ -from lib2to3.fixes.fix_input import context - from django.contrib import messages -from django.contrib.auth import login +from django.contrib.auth import login, get_user_model from django.contrib.auth.mixins import LoginRequiredMixin -from django.contrib.auth.views import LoginView, LogoutView -from django.views.generic import ListView, DetailView -from django.views.generic.edit import CreateView, UpdateView - -from survey.models import Survey -from .mixins import OrganisationRequiredMixin -from .models import Organisation, Project, OrganisationMembership, ProjectOrganisation -from django.shortcuts import get_object_or_404, render -from django.views import View -from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm -from django.contrib.auth import login from django.contrib.auth.views import ( + LoginView, + LogoutView, PasswordResetView, PasswordResetDoneView, PasswordResetConfirmView, PasswordResetCompleteView, ) -from django.shortcuts import redirect, render -from django.urls import reverse_lazy -from django.contrib import messages -from django.contrib.auth import get_user_model -from django.core.exceptions import ObjectDoesNotExist +from django.views.generic import ListView, DetailView +from django.views.generic.edit import CreateView, UpdateView +from django.shortcuts import get_object_or_404, render, redirect +from django.views import View +from django.urls import reverse_lazy, reverse +from django.core.exceptions import PermissionDenied from django.db.models import Count -from .permissions import ( - can_create_projects, - can_view_project, - can_edit_project, - get_project_permissions, -) -from .services import ( - OrganisationService, - ProjectService -) +from survey.models import Survey +from .mixins import OrganisationRequiredMixin +from .models import Organisation, Project, OrganisationMembership, ProjectOrganisation +from .forms import ManagerSignupForm, ManagerLoginForm, UserProfileForm +from .services import project_service, organisation_service from .constants import ROLE_ADMIN, ROLE_PROJECT_MANAGER -from django.core.exceptions import PermissionDenied -from django.urls import reverse -from django.shortcuts import get_object_or_404 - User = get_user_model() @@ -80,7 +61,6 @@ class HomeView(LoginRequiredMixin, View): login_url = "login" def get(self, request): - return render(request, self.template_name, {}) @@ -98,7 +78,6 @@ def form_valid(self, form): return super().form_valid(form) def form_invalid(self, form): - print(form.errors) # Output form errors to the console messages.error( self.request, "There was an error updating your profile. Please try again." ) @@ -125,10 +104,6 @@ class CustomPasswordResetCompleteView(PasswordResetCompleteView): template_name = "home/password_reset_complete.html" -# class PasswordResetExpiredView(TemplateView): # leave for now -# template_name = 'home/password_reset_expired.html' - - class MyOrganisationView(LoginRequiredMixin, OrganisationRequiredMixin, ListView): template_name = "organisation/organisation.html" context_object_name = "projects" @@ -136,10 +111,10 @@ class MyOrganisationView(LoginRequiredMixin, OrganisationRequiredMixin, ListView def setup(self, request, *args, **kwargs): super().setup(request, *args, **kwargs) - self.organisation = OrganisationService.get_user_organisation(request.user) + self.organisation = organisation_service.get_user_organisation(request.user) def get_queryset(self): - return OrganisationService.get_organisation_projects(self.organisation) + return organisation_service.get_organisation_projects(self.organisation) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -151,11 +126,14 @@ def get_context_data(self, **kwargs): context.update( { "organisation": self.organisation, - "can_edit": get_project_permissions(user, projects), - "can_create": can_create_projects(user), + "can_edit": { + project.id: project_service.can_edit(user, project) + for project in projects + }, + "can_create": project_service.can_create(user), "is_admin": user_role == ROLE_ADMIN, "is_project_manager": user_role == ROLE_PROJECT_MANAGER, - "project_orgs": OrganisationService.get_user_accessible_organisations( + "project_orgs": organisation_service.get_user_accessible_organisations( projects, user ), } @@ -173,14 +151,19 @@ def get_success_url(self): return reverse_lazy("myorganisation") def form_valid(self, form): - super().form_valid(form) - OrganisationService.add_user_to_organisation( - user=self.request.user, - organisation=self.object, - role=ROLE_ADMIN, - added_by=self.request.user, - ) - return redirect("myorganisation") + try: + organisation = organisation_service.create_organisation( + user=self.request.user, + name=form.cleaned_data["name"], + description=form.cleaned_data["description"], + ) + self.object = organisation + return redirect(self.get_success_url()) + except PermissionDenied: + messages.error( + self.request, "You don't have permission to create organisations." + ) + return redirect("home") class ProjectView(LoginRequiredMixin, ListView): @@ -195,7 +178,7 @@ def dispatch(self, request, *args, **kwargs): messages.error(request, "Project not found.") return redirect("myorganisation") - if not can_view_project(request.user, self.project): + if not project_service.can_view(request.user, self.project): messages.error( request, f"You do not have permission to view the project {self.project.name}.", @@ -214,7 +197,8 @@ def get_context_data(self, **kwargs): context.update( { "project": project, - "can_create": can_edit_project(user, project), + "can_create": project_service.can_edit(user, project), + "permission": project_service.get_user_permission(user, project), } ) @@ -231,10 +215,10 @@ def get_context_data(self, **kwargs): organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) context["organisation"] = organisation - if not can_create_projects(self.request.user): + if not project_service.can_create(self.request.user): messages.error( self.request, - "You don't have permissions to create projects in this organisation.", + "You don't have permission to create projects in this organisation.", ) return redirect("myorganisation") @@ -244,18 +228,18 @@ def get_success_url(self): return self.object.get_absolute_url() def form_valid(self, form): - form.instance.created_by = self.request.user - result = super().form_valid(form) - - organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) - ProjectService.link_project_to_organisation( - project=self.object, - organisation=organisation, - user=self.request.user, - permission="EDIT", # Project creators get edit permission by default - ) - - return result + try: + organisation = Organisation.objects.get(id=self.kwargs["organisation_id"]) + project = project_service.create_project( + user=self.request.user, + name=form.cleaned_data["name"], + organisation=organisation, + ) + self.object = project + return redirect(self.get_success_url()) + except PermissionDenied: + messages.error(self.request, "Permission denied") + return redirect("myorganisation") class ProjectEditView(LoginRequiredMixin, UpdateView): @@ -272,7 +256,7 @@ def get_object(self, queryset=None): id=self.kwargs["project_id"], ) - if not can_edit_project(self.request.user, project): + if not project_service.can_edit(self.request.user, project): messages.error( self.request, "You don't have permission to edit this project." ) @@ -284,13 +268,15 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) user = self.request.user - # Get all organisations user has access to - project_orgs = OrganisationService.get_user_accessible_organisations( + project_orgs = organisation_service.get_user_accessible_organisations( [self.object], user ).get(self.object.id, []) # Get user's roles across organisations - user_roles = {org.id: org.get_user_role(user) for org in project_orgs} + user_roles = { + org.id: organisation_service.get_user_role(user, org) + for org in project_orgs + } context.update( { @@ -310,8 +296,15 @@ def get_success_url(self): return reverse("myorganisation") def form_valid(self, form): - response = super().form_valid(form) - messages.success( - self.request, f"Project {self.object.name} has been updated successfully." - ) - return response + try: + project_service.update_project( + user=self.request.user, project=self.object, data=form.cleaned_data + ) + messages.success( + self.request, + f"Project {self.object.name} has been updated successfully.", + ) + return redirect(self.get_success_url()) + except PermissionDenied: + messages.error(self.request, "Permission denied") + return redirect("myorganisation") From ade4afcef31031ead9fd2cb2faf8ac01c5d04f32 Mon Sep 17 00:00:00 2001 From: Yuliang Weng <59968766+yld-weng@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:04:49 +0000 Subject: [PATCH 70/79] fix: survey counts --- home/services/__init__.py | 4 ++-- home/services/organisation.py | 6 +++--- home/views.py | 9 +++++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/home/services/__init__.py b/home/services/__init__.py index 2d7e82e..338baa7 100644 --- a/home/services/__init__.py +++ b/home/services/__init__.py @@ -1,4 +1,4 @@ -from .base import BaseService +from .base import BasePermissionService from .project import ProjectService from .organisation import OrganisationService @@ -7,7 +7,7 @@ organisation_service = OrganisationService() __all__ = [ - 'BaseService', + 'BasePermissionService', 'ProjectService', 'OrganisationService', 'project_service', diff --git a/home/services/organisation.py b/home/services/organisation.py index a7b3504..5e2f968 100644 --- a/home/services/organisation.py +++ b/home/services/organisation.py @@ -62,7 +62,7 @@ def get_user_organisation_ids(self, user: User) -> Set[int]: def get_user_accessible_organisations( self, projects: QuerySet[Project], user: User ) -> Dict[int, List[Organisation]]: - """Get organisations for each project that user is member of""" + """Get organisations user has access to for each project""" user_org_ids = self.get_user_organisation_ids(user) return { project.id: [ @@ -139,10 +139,10 @@ def get_organisation_projects( if with_metrics: projects = projects.annotate( - survey_count=Count("survey"), + survey_count=Count("survey__id", distinct=True), manager_count=Count("projectmanagerpermission", distinct=True), ) - + return projects @requires_permission("view") diff --git a/home/views.py b/home/views.py index 26c51ff..9bd2c88 100644 --- a/home/views.py +++ b/home/views.py @@ -114,7 +114,9 @@ def setup(self, request, *args, **kwargs): self.organisation = organisation_service.get_user_organisation(request.user) def get_queryset(self): - return organisation_service.get_organisation_projects(self.organisation) + return organisation_service.get_organisation_projects( + self.organisation + ).order_by("-created_on") def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -187,7 +189,10 @@ def dispatch(self, request, *args, **kwargs): return super().dispatch(request, *args, **kwargs) def get_queryset(self): - return Survey.objects.filter(project_id=self.kwargs["project_id"]) + # Django requires consistent ordering for pagination + return Survey.objects.filter(project_id=self.kwargs["project_id"]).order_by( + "-id" + ) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) From 5c00577fe9ca04f3b79fcde06ae4d4e44943a76d Mon Sep 17 00:00:00 2001 From: f-allian Date: Fri, 17 Jan 2025 13:02:28 +0000 Subject: [PATCH 71/79] feat: search-bar functionality for projects and surveys --- home/forms.py | 10 +++ home/templates/components/search_bar.html | 15 +++++ home/templates/organisation/organisation.html | 27 +++++--- home/templates/projects/project.html | 4 ++ home/templatetags/search_tags.py | 18 ++++++ home/views.py | 61 +++++++++++-------- 6 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 home/templates/components/search_bar.html create mode 100644 home/templatetags/search_tags.py diff --git a/home/forms.py b/home/forms.py index aed3f83..36ec907 100644 --- a/home/forms.py +++ b/home/forms.py @@ -66,3 +66,13 @@ def save(self, commit=True): return user + +class SearchBarForm(forms.Form): + q = forms.CharField(required=False, + widget=forms.TextInput(attrs={ + 'class': 'form-control', + 'placeholder': 'Search...', + 'aria-label': 'Search', + 'name': 'q' + }) + ) diff --git a/home/templates/components/search_bar.html b/home/templates/components/search_bar.html new file mode 100644 index 0000000..3022b4e --- /dev/null +++ b/home/templates/components/search_bar.html @@ -0,0 +1,15 @@ + + +
+ {% for key, value in current_params.items %} + {% if key != 'q' and key != 'page' %} + + {% endif %} + {% endfor %} + {{ search_form.q }} + + {% if current_search %} + Clear + {% endif %} +
+ \ No newline at end of file diff --git a/home/templates/organisation/organisation.html b/home/templates/organisation/organisation.html index e83d435..990569a 100644 --- a/home/templates/organisation/organisation.html +++ b/home/templates/organisation/organisation.html @@ -2,17 +2,24 @@ {% load project_filters %} {% block content %}
- +
+ +
+ {% load search_tags %} + {% search_bar placeholder="Search projects..." %} +
+
+

{{ organisation.name }}

{{ organisation.description }}

diff --git a/home/templates/projects/project.html b/home/templates/projects/project.html index 7e213f9..c44dbae 100644 --- a/home/templates/projects/project.html +++ b/home/templates/projects/project.html @@ -14,6 +14,10 @@ +
+ {% load search_tags %} + {% search_bar placeholder="Search surveys..." %} +

{{ project.name }}

diff --git a/home/templatetags/search_tags.py b/home/templatetags/search_tags.py new file mode 100644 index 0000000..479f55f --- /dev/null +++ b/home/templatetags/search_tags.py @@ -0,0 +1,18 @@ +from django import template +from ..forms import SearchBarForm + +register = template.Library() + + +@register.inclusion_tag('components/search_bar.html', takes_context=True) +def search_bar(context, placeholder="Search...", search_url=None): + request = context['request'] + form = SearchBarForm(request.GET or None) + form.fields['q'].widget.attrs['placeholder'] = placeholder + + return { + 'search_form': form, + 'current_search': request.GET.get('q', ''), + 'search_url': search_url or request.path, + 'current_params': request.GET.copy() + } \ No newline at end of file diff --git a/home/views.py b/home/views.py index 9bd2c88..cedc95f 100644 --- a/home/views.py +++ b/home/views.py @@ -15,7 +15,7 @@ from django.views import View from django.urls import reverse_lazy, reverse from django.core.exceptions import PermissionDenied -from django.db.models import Count +from django.db.models import Count, Q from survey.models import Survey from .mixins import OrganisationRequiredMixin @@ -114,33 +114,38 @@ def setup(self, request, *args, **kwargs): self.organisation = organisation_service.get_user_organisation(request.user) def get_queryset(self): - return organisation_service.get_organisation_projects( + queryset = organisation_service.get_organisation_projects( self.organisation - ).order_by("-created_on") + ) + + search_query = self.request.GET.get('q') + if search_query: + queryset = queryset.filter( + Q(name__icontains=search_query) + ) + + return queryset.order_by("-created_on") def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) user = self.request.user projects = context["projects"] - user_role = self.organisation.get_user_role(user) - context.update( - { - "organisation": self.organisation, - "can_edit": { - project.id: project_service.can_edit(user, project) - for project in projects - }, - "can_create": project_service.can_create(user), - "is_admin": user_role == ROLE_ADMIN, - "is_project_manager": user_role == ROLE_PROJECT_MANAGER, - "project_orgs": organisation_service.get_user_accessible_organisations( - projects, user - ), - } - ) - + context.update({ + "organisation": self.organisation, + "can_edit": { + project.id: project_service.can_edit(user, project) + for project in projects + }, + "can_create": project_service.can_create(user), + "is_admin": user_role == ROLE_ADMIN, + "is_project_manager": user_role == ROLE_PROJECT_MANAGER, + "project_orgs": organisation_service.get_user_accessible_organisations( + projects, user + ), + "current_search": self.request.GET.get('q', '') + }) return context @@ -189,10 +194,17 @@ def dispatch(self, request, *args, **kwargs): return super().dispatch(request, *args, **kwargs) def get_queryset(self): + queryset = Survey.objects.filter(project_id=self.kwargs["project_id"]) + + # Add search if query exists + search_query = self.request.GET.get('q') + if search_query: + queryset = queryset.filter( + Q(name__icontains=search_query) # Only search by survey name + ) + # Django requires consistent ordering for pagination - return Survey.objects.filter(project_id=self.kwargs["project_id"]).order_by( - "-id" - ) + return queryset.order_by("-id") def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -204,6 +216,7 @@ def get_context_data(self, **kwargs): "project": project, "can_create": project_service.can_edit(user, project), "permission": project_service.get_user_permission(user, project), + "current_search": self.request.GET.get('q', '') } ) @@ -312,4 +325,4 @@ def form_valid(self, form): return redirect(self.get_success_url()) except PermissionDenied: messages.error(self.request, "Permission denied") - return redirect("myorganisation") + return redirect("myorganisation") \ No newline at end of file From 136b764ecd0431ec8fc4cb7abfd7c519f9033296 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:52:49 +0000 Subject: [PATCH 72/79] Update CONTRIBUTING.md --- CONTRIBUTING.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d880a0c..d4b3b48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,9 +4,17 @@ We welcome contributions to SORT! This document outlines the guidelines for cont # Getting Started -... +## Organising work -https://www.conventionalcommits.org/en/v1.0.0/#summary +Please use the [Kanban board](https://github.com/orgs/RSE-Sheffield/projects/19) to assign tasks. + +# Making changes + +1. [Raise an issue](https://github.com/RSE-Sheffield/SORT/issues/new?template=Blank+issue) clearly describing the problem or user requirements; +2. [Create a branch](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/creating-a-branch-for-an-issue) that is associated with that issue. It can be helpful to prefix the branch name to match the type of changes e.g. `feat/123-my-feature` for features or `docs/my-guide` for documentation, etc. +3. In that branch, make changes that aim to resolve that issue; +4. Create a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) (PR) while the changes are being designed; +5. When ready, mark the PR "Ready for review" and request for reviewers to look at the proposed changes; # Code of Conduct From 4f3c8213b3b0de83bce1a3b0d282943ca35e3434 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:55:10 +0000 Subject: [PATCH 73/79] Revert SORT/settings.py --- SORT/settings.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/SORT/settings.py b/SORT/settings.py index 6bc1024..2eef0c0 100644 --- a/SORT/settings.py +++ b/SORT/settings.py @@ -20,6 +20,7 @@ # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent + # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ @@ -31,6 +32,7 @@ ALLOWED_HOSTS = [] + # Application definition INSTALLED_APPS = [ @@ -80,6 +82,7 @@ WSGI_APPLICATION = "SORT.wsgi.application" + # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases @@ -113,18 +116,19 @@ }, ] + # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ -LANGUAGE_CODE = "en-gb" +LANGUAGE_CODE = "en-us" TIME_ZONE = "UTC" -# https://docs.djangoproject.com/en/5.1/topics/i18n/translation/ -USE_I18N = False +USE_I18N = True USE_TZ = True + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.1/howto/static-files/ @@ -132,6 +136,7 @@ STATICFILES_DIRS = [BASE_DIR / "static"] + # Default primary key field type # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field @@ -140,6 +145,7 @@ LOGIN_REDIRECT_URL = "/" LOGOUT_REDIRECT_URL = "/" + # FA: End session when the browser is closed SESSION_EXPIRE_AT_BROWSER_CLOSE = True @@ -160,7 +166,7 @@ "127.0.0.1", # ... ] -AUTH_USER_MODEL = 'home.User' # FA: replace username with email as unique identifiers +AUTH_USER_MODEL = 'home.User' # FA: replace username with email as unique identifiers # FA: for production: From 411d334d7c08e4939d1423eb08c38394c451fd0b Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:05:10 +0000 Subject: [PATCH 74/79] Add note on environments --- CONTRIBUTING.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4b3b48..e7b6fc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,12 +10,38 @@ Please use the [Kanban board](https://github.com/orgs/RSE-Sheffield/projects/19) # Making changes +## Proposing changes + 1. [Raise an issue](https://github.com/RSE-Sheffield/SORT/issues/new?template=Blank+issue) clearly describing the problem or user requirements; 2. [Create a branch](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/creating-a-branch-for-an-issue) that is associated with that issue. It can be helpful to prefix the branch name to match the type of changes e.g. `feat/123-my-feature` for features or `docs/my-guide` for documentation, etc. 3. In that branch, make changes that aim to resolve that issue; 4. Create a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) (PR) while the changes are being designed; 5. When ready, mark the PR "Ready for review" and request for reviewers to look at the proposed changes; +# Environments + +There are two main environments: + +- Development (the `dev` branch and the `sort-web-dev` virtual machine) +- Production (the `main` branch and the `sort-web-app` virtual machine) + +Any proposed changes should be proposed in pull requests that would be merged into the `dev` branch. + +```mermaid +graph LR + subgraph Develop + A[Commit to develop branch] + B(Feature branch) + C{Merge feature branch into develop} + end + subgraph Main + D[Main branch] + end + A --> B + B --> C {approved} + C --> D {Merge develop into main} +``` + # Code of Conduct We expect all contributors to follow the SORT [Code of Conduct](CODE_OF_CONDUCT.md). From 05c6cf0522050b45212f12fd78044c250bbc015c Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:08:40 +0000 Subject: [PATCH 75/79] Fix mermaid diag --- CONTRIBUTING.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7b6fc3..61419f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,17 +29,19 @@ Any proposed changes should be proposed in pull requests that would be merged in ```mermaid graph LR - subgraph Develop - A[Commit to develop branch] - B(Feature branch) - C{Merge feature branch into develop} + subgraph Development + A(Feature branch) + B{Approve?} + C[Merge feature branch into dev] + D{Approve?} end - subgraph Main - D[Main branch] + subgraph Production + E[Main branch] end A --> B - B --> C {approved} - C --> D {Merge develop into main} + B -- Yes --> C + C --> D + D -- Yes --> E ``` # Code of Conduct From 0b5827ac2ff66d5e584bde05ff2b7d5e5e99fe8b Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:09:06 +0000 Subject: [PATCH 76/79] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61419f5..f3dae7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,13 +29,13 @@ Any proposed changes should be proposed in pull requests that would be merged in ```mermaid graph LR - subgraph Development + subgraph Development environment A(Feature branch) B{Approve?} C[Merge feature branch into dev] D{Approve?} end - subgraph Production + subgraph Production environment E[Main branch] end A --> B From 5085ee4216ff536448fc29dc4f3a7c19a0925189 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:25:55 +0000 Subject: [PATCH 77/79] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3dae7c..3129a06 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ Please use the [Kanban board](https://github.com/orgs/RSE-Sheffield/projects/19) ## Proposing changes 1. [Raise an issue](https://github.com/RSE-Sheffield/SORT/issues/new?template=Blank+issue) clearly describing the problem or user requirements; -2. [Create a branch](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/creating-a-branch-for-an-issue) that is associated with that issue. It can be helpful to prefix the branch name to match the type of changes e.g. `feat/123-my-feature` for features or `docs/my-guide` for documentation, etc. +2. [Create a branch](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/creating-a-branch-for-an-issue) that is associated with that issue. It can be helpful to prefix the branch name to match the type of changes e.g. `feat/123-my-feature` for features or `docs/my-guide` for documentation, etc. See [Semantic branch names](https://damiandabrowski.medium.com/semantic-branch-names-and-commit-messages-3ac38a6fcbb6). 3. In that branch, make changes that aim to resolve that issue; 4. Create a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) (PR) while the changes are being designed; 5. When ready, mark the PR "Ready for review" and request for reviewers to look at the proposed changes; From d8995d6989b785cffc516405a5ccb7358e9608c0 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:43:06 +0000 Subject: [PATCH 78/79] Add commit diagram --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3129a06..6970ce8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,20 @@ graph LR D -- Yes --> E ``` +so the commit history would look something like this: + +```mermaid +gitGraph + commit id: "Initial commit" + branch dev + branch feat/my-feature + commit id: "Work on feature branch" + checkout dev + merge feat/my-feature id: "Merge into dev" + checkout main + merge dev id: "Merge dev into main" +``` + # Code of Conduct We expect all contributors to follow the SORT [Code of Conduct](CODE_OF_CONDUCT.md). From e2cb202ff0dff256ee894f6f3c8897e69d896e36 Mon Sep 17 00:00:00 2001 From: Joe Heffer <60133133+Joe-Heffer-Shef@users.noreply.github.com> Date: Mon, 20 Jan 2025 13:44:04 +0000 Subject: [PATCH 79/79] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6970ce8..a84dcea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,13 +18,15 @@ Please use the [Kanban board](https://github.com/orgs/RSE-Sheffield/projects/19) 4. Create a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests) (PR) while the changes are being designed; 5. When ready, mark the PR "Ready for review" and request for reviewers to look at the proposed changes; -# Environments +## Environments There are two main environments: - Development (the `dev` branch and the `sort-web-dev` virtual machine) - Production (the `main` branch and the `sort-web-app` virtual machine) +## Change process + Any proposed changes should be proposed in pull requests that would be merged into the `dev` branch. ```mermaid