From 74035abcb2fff16642674c783d5f126ffc33ee46 Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Mon, 20 Sep 2021 22:27:32 -0300 Subject: [PATCH 1/7] Django user registration using kafka --- common/error/handling.py | 7 +- .../login => common/utils}/__init__.py | 0 common/utils/validation.py | 7 + docker-compose.yml | 48 ------- services/gateway/connections/login/apps.py | 4 - services/gateway/connections/login/urls.py | 8 -- services/gateway/connections/login/views.py | 22 ---- services/gateway/connections/users/views.py | 5 +- services/gateway/gateway/settings.py | 1 - services/gateway/gateway/urls.py | 1 - services/login/Dockerfile | 17 --- services/login/createAndConfigure.py | 23 ---- services/login/entrypoint.sh | 6 - services/login/login/__main__.py | 11 -- services/login/login/asgi.py | 7 - services/login/login/settings.py | 122 ------------------ services/login/login/urls.py | 8 -- services/login/login/wsgi.py | 7 - services/login/manage.py | 34 ----- services/login/modules/__init__.py | 0 services/login/modules/api/__init__.py | 0 services/login/modules/api/actions.py | 17 --- services/login/modules/api/agents.py | 18 --- services/login/modules/api/apps.py | 4 - services/login/modules/api/topics.py | 5 - services/login/modules/api/urls.py | 5 - .../login/modules/kafka_handler/__init__.py | 0 services/login/modules/kafka_handler/app.py | 32 ----- services/login/modules/kafka_handler/apps.py | 4 - .../kafka_handler/migrations/__init__.py | 0 services/login/requirements.txt | 1 - services/login/setup.py | 41 ------ services/users/modules/api/actions.py | 31 ++++- services/users/modules/api/urls.py | 5 - .../modules/authorization}/__init__.py | 0 services/users/modules/authorization/apps.py | 4 + .../users/modules/authorization/services.py | 15 +++ .../kafka_handler/migrations/__init__.py | 0 services/users/users/settings.py | 5 +- 39 files changed, 61 insertions(+), 464 deletions(-) rename {services/gateway/connections/login => common/utils}/__init__.py (100%) create mode 100644 common/utils/validation.py delete mode 100644 services/gateway/connections/login/apps.py delete mode 100644 services/gateway/connections/login/urls.py delete mode 100644 services/gateway/connections/login/views.py delete mode 100644 services/login/Dockerfile delete mode 100644 services/login/createAndConfigure.py delete mode 100755 services/login/entrypoint.sh delete mode 100644 services/login/login/__main__.py delete mode 100644 services/login/login/asgi.py delete mode 100644 services/login/login/settings.py delete mode 100644 services/login/login/urls.py delete mode 100644 services/login/login/wsgi.py delete mode 100755 services/login/manage.py delete mode 100644 services/login/modules/__init__.py delete mode 100644 services/login/modules/api/__init__.py delete mode 100644 services/login/modules/api/actions.py delete mode 100644 services/login/modules/api/agents.py delete mode 100644 services/login/modules/api/apps.py delete mode 100644 services/login/modules/api/topics.py delete mode 100644 services/login/modules/api/urls.py delete mode 100644 services/login/modules/kafka_handler/__init__.py delete mode 100644 services/login/modules/kafka_handler/app.py delete mode 100644 services/login/modules/kafka_handler/apps.py delete mode 100644 services/login/modules/kafka_handler/migrations/__init__.py delete mode 100644 services/login/requirements.txt delete mode 100644 services/login/setup.py delete mode 100644 services/users/modules/api/urls.py rename services/{login/login => users/modules/authorization}/__init__.py (100%) create mode 100644 services/users/modules/authorization/apps.py create mode 100644 services/users/modules/authorization/services.py delete mode 100644 services/users/modules/kafka_handler/migrations/__init__.py diff --git a/common/error/handling.py b/common/error/handling.py index d15efb8..fe873fa 100644 --- a/common/error/handling.py +++ b/common/error/handling.py @@ -1,10 +1,11 @@ from common.models.message import Message -def handleError(event: Message, information="", where=""): +def handleError(event: Message, information="", where="", status=400): event.error = True event.data = { - "information":information, - "where":where + "information": information, + "where": where, + "status": status, } def checkError(event: Message, where): diff --git a/services/gateway/connections/login/__init__.py b/common/utils/__init__.py similarity index 100% rename from services/gateway/connections/login/__init__.py rename to common/utils/__init__.py diff --git a/common/utils/validation.py b/common/utils/validation.py new file mode 100644 index 0000000..97d9d9e --- /dev/null +++ b/common/utils/validation.py @@ -0,0 +1,7 @@ + +def is_key_null(obj, key): + if key in obj: + if obj[key] != "": + return False + + return True \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 75ba723..f970546 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,54 +27,6 @@ services: - kafka - postgres - login: - image: farmacia-solidaria/login:latest - restart: unless-stopped - environment: - - SERVICE_ADDR=0.0.0.0 - - SERVICE_PORT=8001 - - DEBUG=TRUE - - SECRET_KEY=secret_key - - DATABASE_USERNAME=root - - DATABASE_PASSWORD=root - - DATABASE_HOST=postgres - - DATABASE_PORT=5432 - depends_on: - - kafka - - postgres - - products: - image: farmacia-solidaria/products:latest - restart: unless-stopped - environment: - - SERVICE_ADDR=0.0.0.0 - - SERVICE_PORT=8001 - - DEBUG=TRUE - - SECRET_KEY=secret_key - - DATABASE_USERNAME=root - - DATABASE_PASSWORD=root - - DATABASE_HOST=postgres - - DATABASE_PORT=5432 - depends_on: - - kafka - - postgres - - report: - image: farmacia-solidaria/report:latest - restart: unless-stopped - environment: - - SERVICE_ADDR=0.0.0.0 - - SERVICE_PORT=8001 - - DEBUG=TRUE - - SECRET_KEY=secret_key - - DATABASE_USERNAME=root - - DATABASE_PASSWORD=root - - DATABASE_HOST=postgres - - DATABASE_PORT=5432 - depends_on: - - kafka - - postgres - users: image: farmacia-solidaria/users:latest restart: unless-stopped diff --git a/services/gateway/connections/login/apps.py b/services/gateway/connections/login/apps.py deleted file mode 100644 index 0e46153..0000000 --- a/services/gateway/connections/login/apps.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.apps import AppConfig - -class LoginConfig(AppConfig): - name = 'connections.login' diff --git a/services/gateway/connections/login/urls.py b/services/gateway/connections/login/urls.py deleted file mode 100644 index 86ebc71..0000000 --- a/services/gateway/connections/login/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.contrib import admin -from django.urls import path - -from .views import LoginViewset - -urlpatterns = [ - path('login/', LoginViewset.as_view()), -] diff --git a/services/gateway/connections/login/views.py b/services/gateway/connections/login/views.py deleted file mode 100644 index eb9ed03..0000000 --- a/services/gateway/connections/login/views.py +++ /dev/null @@ -1,22 +0,0 @@ -from django.http.response import JsonResponse -from rest_framework import viewsets, status -from rest_framework.response import Response -from rest_framework.views import APIView -from common.kafka.send import send_and_wait_message - -class LoginViewset(APIView): - - def post(self, request, action): - - data = send_and_wait_message( - service="login", - action=action, - data={ - "username": request.data['username'], - "password": request.data["password"] - }, - filter=True, - suppress_errors=True - ) - - return Response(data, status=status.HTTP_200_OK) \ No newline at end of file diff --git a/services/gateway/connections/users/views.py b/services/gateway/connections/users/views.py index de547b6..11b7b07 100644 --- a/services/gateway/connections/users/views.py +++ b/services/gateway/connections/users/views.py @@ -16,4 +16,7 @@ def post(self, request, action): suppress_errors=True ) - return Response(data, status=status.HTTP_200_OK) \ No newline at end of file + if data: + return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) + + return Response(status=status.HTTP_408_REQUEST_TIMEOUT) \ No newline at end of file diff --git a/services/gateway/gateway/settings.py b/services/gateway/gateway/settings.py index fd4f39e..deb1c21 100644 --- a/services/gateway/gateway/settings.py +++ b/services/gateway/gateway/settings.py @@ -45,7 +45,6 @@ ] CONNECTIONS = [ - 'connections.login', 'connections.users', ] diff --git a/services/gateway/gateway/urls.py b/services/gateway/gateway/urls.py index d4108ef..0ba6280 100644 --- a/services/gateway/gateway/urls.py +++ b/services/gateway/gateway/urls.py @@ -17,6 +17,5 @@ from django.urls import path, include urlpatterns = [ - path('api/', include('connections.login.urls')), path('api/', include('connections.users.urls')), ] diff --git a/services/login/Dockerfile b/services/login/Dockerfile deleted file mode 100644 index db35e06..0000000 --- a/services/login/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9 -ENV PYTHONUNBUFFERED 1 - -WORKDIR /app - -RUN apt-get update && \ - apt-get install -y --no-install-recommends python-dev librocksdb-dev build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev - -COPY setup.py /app -COPY requirements.txt /app - -RUN pip install -r requirements.txt - -COPY . /app -RUN pip install . - -CMD [ "bash", "/app/entrypoint.sh" ] \ No newline at end of file diff --git a/services/login/createAndConfigure.py b/services/login/createAndConfigure.py deleted file mode 100644 index 2b8c99f..0000000 --- a/services/login/createAndConfigure.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import psycopg2 -from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT - -conn = psycopg2.connect( - dbname = 'root', - user = os.getenv("DATABASE_USERNAME"), - password = os.getenv("DATABASE_PASSWORD"), - host = os.getenv("DATABASE_HOST"), -) - -conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - -cur = conn.cursor() - -try: - cur.execute('CREATE DATABASE "fs-login" ') -except: - print("Database already created, starting server") - - -conn.close() -cur.close() \ No newline at end of file diff --git a/services/login/entrypoint.sh b/services/login/entrypoint.sh deleted file mode 100755 index e0c04f2..0000000 --- a/services/login/entrypoint.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -python createAndConfigure.py - -(service-django runserver) & -(service-faust worker -l info) \ No newline at end of file diff --git a/services/login/login/__main__.py b/services/login/login/__main__.py deleted file mode 100644 index b8ae98b..0000000 --- a/services/login/login/__main__.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'login.settings') -from django.core.management import execute_from_command_line # noqa: E402 - -# Override default port for `runserver` command -from django.core.management.commands.runserver import Command as runserver -runserver.default_port = os.getenv("SERVICE_PORT") -runserver.default_addr = os.getenv("SERVICE_ADDR") - -execute_from_command_line(sys.argv) diff --git a/services/login/login/asgi.py b/services/login/login/asgi.py deleted file mode 100644 index 9164e40..0000000 --- a/services/login/login/asgi.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -from django.core.asgi import get_asgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'login.settings') - -application = get_asgi_application() diff --git a/services/login/login/settings.py b/services/login/login/settings.py deleted file mode 100644 index 327fc67..0000000 --- a/services/login/login/settings.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -from pathlib import Path - -BASE_DIR = Path(__file__).resolve().parent.parent - -SECRET_KEY = os.getenv("SECRET_KEY") - -DEBUG = os.getenv("DEBUG") - -ALLOWED_HOSTS = [ - "gateway", - "localhost" -] - -# Kafka and Faust Configuration -KAFKA_BROKER_URL = 'kafka:9092' -FAUST_STORE_URL = 'rocksdb://' - - -LOCAL_APPS = [ - 'modules.kafka_handler', - 'modules.api' -] - -THIRD_PARTY_APPS = [ - 'rest_framework', - 'corsheaders', -] - - -# Application definition -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - *THIRD_PARTY_APPS, - *LOCAL_APPS, - -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'corsheaders.middleware.CorsMiddleware', -] - -ROOT_URLCONF = 'login.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'login.wsgi.application' - - -# Database -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'fs-login', - 'USER': os.getenv("DATABASE_USERNAME"), - 'PASSWORD': os.getenv("DATABASE_PASSWORD"), - 'HOST': os.getenv("DATABASE_HOST"), - 'PORT': os.getenv("DATABASE_PORT"), - } -} - - -# Password validation -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -LANGUAGE_CODE = 'pt-br' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -STATIC_URL = '/static/' - -# Default primary key field type -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/services/login/login/urls.py b/services/login/login/urls.py deleted file mode 100644 index ef05f1f..0000000 --- a/services/login/login/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.contrib import admin -from django.conf.urls import url -from django.urls import path, include - -urlpatterns = [ - # url(r'^admin/', admin.site.urls), - # path("api/", include('modules.api.urls')) -] diff --git a/services/login/login/wsgi.py b/services/login/login/wsgi.py deleted file mode 100644 index 9e7cef8..0000000 --- a/services/login/login/wsgi.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'login.settings') - -application = get_wsgi_application() diff --git a/services/login/manage.py b/services/login/manage.py deleted file mode 100755 index df581f0..0000000 --- a/services/login/manage.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -"""Django's command-line utility for administrative tasks.""" -import os -import sys - - -def main(): - """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'login.settings') - try: - from django.core.management import execute_from_command_line - - import django - django.setup() - - # Override default port for `runserver` command - from django.core.management.commands.runserver import Command as runserver - runserver.default_port = os.getenv("SERVICE_PORT") - runserver.default_addr = os.getenv("SERVICE_ADDR") - - from django.core.management import execute_from_command_line - - - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc - execute_from_command_line(sys.argv) - - -if __name__ == '__main__': - main() diff --git a/services/login/modules/__init__.py b/services/login/modules/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/services/login/modules/api/__init__.py b/services/login/modules/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/services/login/modules/api/actions.py b/services/login/modules/api/actions.py deleted file mode 100644 index 447baa7..0000000 --- a/services/login/modules/api/actions.py +++ /dev/null @@ -1,17 +0,0 @@ -from common.error.handling import handleError -from common.models.message import Message -from common.kafka.actions import ActionHandler - - -actioneer = ActionHandler() - -@actioneer.register -async def auth(message: Message): - message.data["JWToken"] = "fake-token-jwt" - -@actioneer.default -async def default(message: Message): - handleError( - message, - information="Action not implemented", - where="login") diff --git a/services/login/modules/api/agents.py b/services/login/modules/api/agents.py deleted file mode 100644 index fe1e5de..0000000 --- a/services/login/modules/api/agents.py +++ /dev/null @@ -1,18 +0,0 @@ -from common.error.handling import checkError -from faust.types import StreamT -from common.models.message import Message - -from modules.api.actions import actioneer -from modules.api.topics import defaultInTopic, defaultOutTopic -from modules.kafka_handler.app import faustApp - -@faustApp.agent(defaultInTopic, sink=[defaultOutTopic], concurrency=100) -async def defaultAgent(messages: StreamT[Message]): - async for event in messages: - if checkError(event, 'login'): - yield event - - action = actioneer.get_action(event.action) - await action(event) - - yield event \ No newline at end of file diff --git a/services/login/modules/api/apps.py b/services/login/modules/api/apps.py deleted file mode 100644 index 133e903..0000000 --- a/services/login/modules/api/apps.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.apps import AppConfig - -class ApiConfig(AppConfig): - name = 'modules.api' diff --git a/services/login/modules/api/topics.py b/services/login/modules/api/topics.py deleted file mode 100644 index 60f6307..0000000 --- a/services/login/modules/api/topics.py +++ /dev/null @@ -1,5 +0,0 @@ -from modules.kafka_handler.app import faustApp -from common.models.message import Message - -defaultInTopic = faustApp.topic("login-income", value_type=Message) -defaultOutTopic = faustApp.topic("login-outcome", value_type=Message) \ No newline at end of file diff --git a/services/login/modules/api/urls.py b/services/login/modules/api/urls.py deleted file mode 100644 index d2d839f..0000000 --- a/services/login/modules/api/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.urls import path - -urlpatterns = [ - -] diff --git a/services/login/modules/kafka_handler/__init__.py b/services/login/modules/kafka_handler/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/services/login/modules/kafka_handler/app.py b/services/login/modules/kafka_handler/app.py deleted file mode 100644 index ef896e5..0000000 --- a/services/login/modules/kafka_handler/app.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import sys -import faust -from django.conf import settings - -# Append parent directory to syspath so is possible to import other apps here -sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -os.environ.setdefault( - 'DJANGO_SETTINGS_MODULE', - 'login.settings' -) - -faustApp = faust.App( - 'login-kafka_handler', - autodiscover=[ - "modules.api.agents" - ], - origin='modules' -) - -@faustApp.on_configured.connect -def configure_from_settings(app, conf, **kwargs): - conf.broker = "kafka://"+settings.KAFKA_BROKER_URL - conf.store = settings.FAUST_STORE_URL - -def main(): - faustApp.main() - -if __name__ == '__main__': - main() diff --git a/services/login/modules/kafka_handler/apps.py b/services/login/modules/kafka_handler/apps.py deleted file mode 100644 index d8c6e8d..0000000 --- a/services/login/modules/kafka_handler/apps.py +++ /dev/null @@ -1,4 +0,0 @@ -from django.apps import AppConfig - -class KafkaHandlerConfig(AppConfig): - name = 'modules.kafka_handler' diff --git a/services/login/modules/kafka_handler/migrations/__init__.py b/services/login/modules/kafka_handler/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/services/login/requirements.txt b/services/login/requirements.txt deleted file mode 100644 index 945c9b4..0000000 --- a/services/login/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/services/login/setup.py b/services/login/setup.py deleted file mode 100644 index 4be4003..0000000 --- a/services/login/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -from pathlib import Path -from setuptools import find_packages, setup - -requirements = [ - #Django - "Django==3.2.5", - "djangorestframework==3.12.4", - "django-cors-headers==3.5.0", - "psycopg2==2.9.1", - "requests==2.26.0", - - #Kafka - "faust[rocksdb]==1.10.4", - "kafka-python==1.4.7", - "robinhood-aiokafka==1.1.6", - - #Build - "build==0.6.0.post1", -] - -setup( - name="login", - version="0.0.1", - description="Sample Description", - author="Farmácia Solidária Devs", - author_email="dev@dev.com", - url="https://github.com/Farmacia-Solidaria", - platforms=['any'], - packages=find_packages(), - include_package_data=True, - python_requires='>=3.8.0', - keywords=[], - zip_safe=False, - install_requires=requirements, - entry_points={ - 'console_scripts': [ - 'service-django = login.__main__:main', - 'service-faust = modules.kafka_handler.app:main', - ], - }, -) diff --git a/services/users/modules/api/actions.py b/services/users/modules/api/actions.py index 9685546..3e927da 100644 --- a/services/users/modules/api/actions.py +++ b/services/users/modules/api/actions.py @@ -1,14 +1,25 @@ from common.error.error import ActionError -from common.error.handling import handleError from common.models.message import Message from common.kafka.actions import ActionHandler +from common.error.handling import handleError +from common.utils.validation import is_key_null + +import modules.authorization.services as authorization_service actioneer = ActionHandler() + @actioneer.register -async def information(message: Message): +async def auth(message: Message): try: - message.data = "users is working correctly !!" + if ( + is_key_null(message.data, 'username') or + is_key_null(message.data, 'password') + ): + raise ActionError("You need to provide username and password") + + message.data["token"] = "Fake Token" + except ActionError as ex: handleError( message, @@ -17,9 +28,17 @@ async def information(message: Message): ) @actioneer.register -async def teste(message: Message): +async def register(message: Message): try: - message.data = "Teste message" + if ( + is_key_null(message.data, 'username') or + is_key_null(message.data, 'email') or + is_key_null(message.data, 'password') + ): + raise ActionError("You need to provide username and password") + + message.data = await authorization_service.create_user(message.data) + except ActionError as ex: handleError( message, @@ -27,8 +46,6 @@ async def teste(message: Message): where="login" ) - - @actioneer.default async def default(message: Message): handleError( diff --git a/services/users/modules/api/urls.py b/services/users/modules/api/urls.py deleted file mode 100644 index d2d839f..0000000 --- a/services/users/modules/api/urls.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.urls import path - -urlpatterns = [ - -] diff --git a/services/login/login/__init__.py b/services/users/modules/authorization/__init__.py similarity index 100% rename from services/login/login/__init__.py rename to services/users/modules/authorization/__init__.py diff --git a/services/users/modules/authorization/apps.py b/services/users/modules/authorization/apps.py new file mode 100644 index 0000000..57c12b8 --- /dev/null +++ b/services/users/modules/authorization/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class AuthConfig(AppConfig): + name = 'modules.authorization' diff --git a/services/users/modules/authorization/services.py b/services/users/modules/authorization/services.py new file mode 100644 index 0000000..248c80e --- /dev/null +++ b/services/users/modules/authorization/services.py @@ -0,0 +1,15 @@ +from django.contrib.auth.models import User +from django.contrib.auth.forms import UserCreationForm +from asgiref.sync import sync_to_async + +@sync_to_async +def create_user(data): + data['password1'] = data['password'] + data['password2'] = data['password'] + + form = UserCreationForm(data) + if form.is_valid(): + form.save() + return True + + return form.errors diff --git a/services/users/modules/kafka_handler/migrations/__init__.py b/services/users/modules/kafka_handler/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/services/users/users/settings.py b/services/users/users/settings.py index 94edfa7..73c956a 100644 --- a/services/users/users/settings.py +++ b/services/users/users/settings.py @@ -19,7 +19,8 @@ LOCAL_APPS = [ 'modules.kafka_handler', - 'modules.api' + 'modules.api', + 'modules.authorization', ] THIRD_PARTY_APPS = [ @@ -104,7 +105,7 @@ # Internationalization -LANGUAGE_CODE = 'pt-br' +LANGUAGE_CODE = 'en' TIME_ZONE = 'UTC' From 19c7b1a86cf1a66d6c482ac8a06103c2491aa23e Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Fri, 24 Sep 2021 14:14:03 -0300 Subject: [PATCH 2/7] Added secrets as submodule --- .gitmodules | 3 +++ secrets | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 secrets diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..726c0a3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "secrets"] + path = secrets + url = https://github.com/Farmacia-Solidaria/secrets diff --git a/secrets b/secrets new file mode 160000 index 0000000..01d9729 --- /dev/null +++ b/secrets @@ -0,0 +1 @@ +Subproject commit 01d97297607dc013f095ff16edef5ef437c1f71d From 1de0daad4e52f1cc87394a25e55933b9884132c7 Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Fri, 24 Sep 2021 22:28:02 -0300 Subject: [PATCH 3/7] Enabled authentication with JWT #8 --- .default.env | 13 +++++ README.md | 7 ++- common/kafka/actions.py | 34 ++++++++---- common/kafka/builders.py | 5 +- common/kafka/send.py | 30 ++++++++--- common/models/message.py | 1 + common/utils/information.py | 7 +++ configuration/kafka-config/server.properties | 2 +- docker-compose.yml | 24 ++++++--- requirements.dev.txt | 27 ++++++++++ safeBuild.py | 18 ++++--- secrets | 2 +- services/gateway/connections/users/views.py | 54 ++++++++++++++----- services/gateway/gateway/settings.py | 17 +++--- services/users/modules/api/actions.py | 23 ++++++-- services/users/modules/api/agents.py | 2 +- services/users/modules/authorization/forms.py | 19 +++++++ .../users/modules/authorization/models.py | 14 +++++ .../users/modules/authorization/services.py | 35 +++++++++++- services/users/setup.py | 4 ++ services/users/users/settings.py | 2 +- services/users/users/urls.py | 2 +- 22 files changed, 276 insertions(+), 66 deletions(-) create mode 100644 .default.env create mode 100644 common/utils/information.py create mode 100644 services/users/modules/authorization/forms.py create mode 100644 services/users/modules/authorization/models.py diff --git a/.default.env b/.default.env new file mode 100644 index 0000000..5cf3314 --- /dev/null +++ b/.default.env @@ -0,0 +1,13 @@ +# SERVER +DJANGO_SECRET_KEY=secret_key + +# DATABASE +DATABASE_USERNAME=root +DATABASE_PASSWORD=root + +# TOKEN SIGNATURE +PRIVATE_KEY=PRIVATE KEY +PUBLIC_KEY=PUBLIC KEY + +# OTHERS +DEBUG=TRUE \ No newline at end of file diff --git a/README.md b/README.md index a1fbdc4..ed0c2ee 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ ## Installation To install please compile everything using: -``` +```bash python3 safeBuild.py --containers all --update-commons --env dev ``` + +To deploy use: +```bash +docker-compose --env-file ./secrets/.{ENV}.env up +``` \ No newline at end of file diff --git a/common/kafka/actions.py b/common/kafka/actions.py index 393f4de..ef490e5 100644 --- a/common/kafka/actions.py +++ b/common/kafka/actions.py @@ -4,18 +4,29 @@ class ActionHandler(): - actions: dict = {} + actions: dict = { + "post": {}, + "get": {}, + "put": {}, + "patch": {}, + "delete": {}, + "options": {}, + "head": {}, + } default_function: Callable = None on_error: Callable = lambda: False - def register(self, func): + def register(self, method:str = "post"): + def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return func(*args, **kwargs) - - self.actions[func.__name__] = wrapper - return wrapper + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + self.actions[method.lower()][func.__name__] = wrapper + return wrapper + + return decorator def default(self, func): self.default_function = func @@ -34,9 +45,10 @@ def run_action(self, action, *args, **kwargs): return self.on_error(action, ex, *args, **kwargs) - def get_action(self, action): - if action in self.actions: - return self.actions[action] + def get_action(self, method, action): + if method in self.actions: + if action in self.actions[method]: + return self.actions[method][action] if self.default_function != None: return self.default_function diff --git a/common/kafka/builders.py b/common/kafka/builders.py index bf4f835..cbb34c9 100644 --- a/common/kafka/builders.py +++ b/common/kafka/builders.py @@ -7,8 +7,6 @@ import asyncio -loop = asyncio.get_event_loop() - def build_kafka_consumer(topic, timeout_in_seconds=15): return KafkaConsumer( topic, @@ -19,6 +17,9 @@ def build_kafka_consumer(topic, timeout_in_seconds=15): ) def build_async_kafka_consumer(topic, timeout_in_seconds=15): + + loop = asyncio.get_event_loop() + return AIOKafkaConsumer( topic, loop=loop, diff --git a/common/kafka/send.py b/common/kafka/send.py index 7b3a0d4..85f11f6 100644 --- a/common/kafka/send.py +++ b/common/kafka/send.py @@ -1,7 +1,7 @@ from common.error.error import ActionError import uuid import json -import os +import hashlib from common.base import kafkaProducer from common.models.message import Message @@ -28,12 +28,20 @@ async def async_send_and_wait_message(service, action, data, filter=False, suppr return value -def send_and_wait_message(service, action, data, filter=False, suppress_errors=False) -> dict: +def send_and_wait_message(method, service, action, data, filter=False, suppress_errors=False) -> dict: key = str(uuid.uuid1()) - consumer = build_kafka_consumer(service+"-outcome", timeout_in_seconds=2) + consumer = build_kafka_consumer(service+"-outcome", timeout_in_seconds=10) + + data = _treat_sensible_information(data) - send_message(service, action, data, key) + send_message( + service=service, + action=action, + data=data, + method=method, + key=key + ) for event in consumer: value = json.loads(event.value.decode("utf-8")) @@ -43,13 +51,14 @@ def send_and_wait_message(service, action, data, filter=False, suppress_errors=F value = _filter_value(value) return value -def send_message(service, action, data, key=""): +def send_message(service, action, data, method, key=""): finalKey = key if key != "" else str(uuid.uuid1()) message = Message( action=action, + method=method, data=data, - id=finalKey + id=finalKey, ) try: _send_to_topic(service+"-income", finalKey, message.dumps()) @@ -89,4 +98,13 @@ def _filter_value(value): "data": value["data"], } +def _treat_sensible_information(data): + allow_list = ['password'] + + for i in data: + if i in allow_list: + data[i] = hashlib.sha512(str(i).encode()).hexdigest() + + return data + #endregion \ No newline at end of file diff --git a/common/models/message.py b/common/models/message.py index 30994d9..df3f978 100644 --- a/common/models/message.py +++ b/common/models/message.py @@ -4,4 +4,5 @@ class Message(faust.Record, serializer='json'): action: str data: dict id: str + method: str error: bool = False \ No newline at end of file diff --git a/common/utils/information.py b/common/utils/information.py new file mode 100644 index 0000000..7299625 --- /dev/null +++ b/common/utils/information.py @@ -0,0 +1,7 @@ +import os + +def get_public_key(): + return os.environ["PUBLIC_KEY"].replace("\\n", "\n") + +def get_private_key(): + return os.environ["PRIVATE_KEY"].replace("\\n", "\n") \ No newline at end of file diff --git a/configuration/kafka-config/server.properties b/configuration/kafka-config/server.properties index 2b284a1..2a5fc37 100644 --- a/configuration/kafka-config/server.properties +++ b/configuration/kafka-config/server.properties @@ -9,4 +9,4 @@ max.request.size=1048576 sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512 sasl.mechanism.inter.broker.protocol= zookeeper.connect=zookeeper:2181 -log.retention.ms=15000 +log.retention.ms=5000 diff --git a/docker-compose.yml b/docker-compose.yml index f970546..93269d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,14 @@ services: environment: - SERVICE_ADDR=0.0.0.0 - SERVICE_PORT=8000 + - DATABASE_HOST=postgres + - DATABASE_PORT=5432 + - DEBUG + - DJANGO_SECRET_KEY + - DATABASE_USERNAME + - DATABASE_PASSWORD + - PRIVATE_KEY + - PUBLIC_KEY ports: - 8000:8000 depends_on: @@ -33,15 +41,19 @@ services: environment: - SERVICE_ADDR=0.0.0.0 - SERVICE_PORT=8001 - - DEBUG=TRUE - - SECRET_KEY=secret_key - - DATABASE_USERNAME=root - - DATABASE_PASSWORD=root - DATABASE_HOST=postgres - DATABASE_PORT=5432 + - DEBUG + - DJANGO_SECRET_KEY + - DATABASE_USERNAME + - DATABASE_PASSWORD + - PRIVATE_KEY + - PUBLIC_KEY depends_on: - kafka - postgres + ports: + - 8001:8001 postgres: image: postgres @@ -50,8 +62,8 @@ services: - 9000:5432 environment: - POSTGRES_DB=root - - POSTGRES_PASSWORD=root - - POSTGRES_USER=root + - POSTGRES_USER=${DATABASE_USERNAME} + - POSTGRES_PASSWORD=${DATABASE_PASSWORD} volumes: - ./data/postgres-data:/var/lib/postgresql/data diff --git a/requirements.dev.txt b/requirements.dev.txt index 27983eb..b5f8bcb 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -5,9 +5,11 @@ arrow==1.1.1 asgiref==3.4.1 async-timeout==3.0.1 attrs==21.2.0 +backcall==0.2.0 binaryornot==0.4.4 build==0.6.0.post1 certifi==2021.5.30 +cffi==1.14.6 chardet==4.0.0 charset-normalizer==2.0.4 click==7.1.2 @@ -15,47 +17,72 @@ colorclass==2.2.0 colorlog==5.0.1 cookiecutter==1.7.3 croniter==1.0.15 +cryptography==3.4.8 +debugpy==1.4.1 +decorator==5.0.9 Django==3.2.5 django-cors-headers==3.7.0 djangorestframework==3.12.4 dnspython==1.16.0 +docopt==0.6.2 +entrypoints==0.3 eventlet==0.31.1 faust==1.10.4 Flask==2.0.1 greenlet==1.1.1 idna==3.2 +ipykernel==6.4.0 +ipython==7.27.0 +ipython-genutils==0.2.0 itsdangerous==2.0.1 +jedi==0.18.0 Jinja2==3.0.1 jinja2-time==0.2.0 +jupyter-client==7.0.2 +jupyter-core==4.7.1 kafka-python==2.0.2 MarkupSafe==2.0.1 +matplotlib-inline==0.1.3 mode==4.3.2 multidict==5.1.0 mypy-extensions==0.4.3 +nest-asyncio==1.5.1 opentracing==1.3.0 packaging==21.0 +parso==0.8.2 pep517==0.11.0 +pexpect==4.8.0 +pickleshare==0.7.5 +pipreqs==0.4.10 poyo==0.5.0 prompt-toolkit==1.0.14 psycopg2==2.9.1 +ptyprocess==0.7.0 +pycparser==2.20 Pygments==2.10.0 PyInquirer==1.0.3 +PyJWT==2.1.0 pyparsing==2.4.7 python-dateutil==2.8.2 python-rocksdb==0.7.0 python-slugify==5.0.2 pytz==2021.1 +pyzmq==22.2.1 regex==2021.8.3 requests==2.26.0 robinhood-aiokafka==1.1.6 six==1.16.0 sqlparse==0.4.1 +supervisor==4.2.2 terminaltables==3.1.0 text-unidecode==1.3 tomli==1.2.1 +tornado==6.1 +traitlets==5.1.0 typing-extensions==3.10.0.0 urllib3==1.26.6 venusian==1.2.0 wcwidth==0.2.5 Werkzeug==2.0.1 +yarg==0.1.9 yarl==1.6.3 diff --git a/safeBuild.py b/safeBuild.py index 4edf314..5105cd3 100644 --- a/safeBuild.py +++ b/safeBuild.py @@ -12,6 +12,12 @@ parser.add_argument("--update", dest="update", const=True, default=False, action="store_const", help="Update container if running") parser.add_argument("--update-commons", dest="common", const=True, default=False, action="store_const", help="Update commons") +enviroments = [ + "development", + "production", + "staging" +] + def main(args): containers = listdir("services") toCompile = [] @@ -19,16 +25,12 @@ def main(args): selectedBefore = [i for i in file.read().splitlines()] options = [] - if args.env not in ['dev', 'prod', 'staging']: + if args.env not in enviroments: options.append({ 'type': 'list', 'name': 'env', 'message': "In what enviroment?", - 'choices': [ - "dev", - "prod", - "staging" - ], + 'choices': enviroments, }) if args.only: @@ -58,7 +60,7 @@ def main(args): for container in toCompile: command = f""" cd services/{container} && - docker build -t farmacia-solidaria/{container}:{args.env} . && + docker build --network=host -t farmacia-solidaria/{container}:{args.env} . && docker tag farmacia-solidaria/{container}:{args.env} farmacia-solidaria/{container}:latest """ @@ -75,7 +77,7 @@ def main(args): command = f""" docker-compose stop {container} && docker-compose kill {container} && - docker-compose up -d --no-deps {container} + docker-compose --env-file ./secrets/.{args.env}.env up -d --no-deps {container} """ print(f"\nKilling and updating container {container}:{args.env}:") diff --git a/secrets b/secrets index 01d9729..b1e8edc 160000 --- a/secrets +++ b/secrets @@ -1 +1 @@ -Subproject commit 01d97297607dc013f095ff16edef5ef437c1f71d +Subproject commit b1e8edc1d35eb494f11e2079ef14f6262852d6ed diff --git a/services/gateway/connections/users/views.py b/services/gateway/connections/users/views.py index 11b7b07..62dd530 100644 --- a/services/gateway/connections/users/views.py +++ b/services/gateway/connections/users/views.py @@ -7,16 +7,46 @@ class UserViewset(APIView): def post(self, request, action): + data = send_and_wait_message( + service="users", + method="post", + action=action, + data=request.data, + filter=True, + suppress_errors=True + ) + + if data: + return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) + + return Response(status=status.HTTP_408_REQUEST_TIMEOUT) - data = send_and_wait_message( - service="users", - action=action, - data=request.data, - filter=True, - suppress_errors=True - ) - - if data: - return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) - - return Response(status=status.HTTP_408_REQUEST_TIMEOUT) \ No newline at end of file + def get(self, request, action): + data = send_and_wait_message( + service="users", + method="get", + action=action, + data=request.data, + filter=True, + suppress_errors=True + ) + + if data: + return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) + + return Response(status=status.HTTP_408_REQUEST_TIMEOUT) + + def delete(self, request, action): + data = send_and_wait_message( + service="users", + method="delete", + action=action, + data=request.data, + filter=True, + suppress_errors=True + ) + + if data: + return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) + + return Response(status=status.HTTP_408_REQUEST_TIMEOUT) \ No newline at end of file diff --git a/services/gateway/gateway/settings.py b/services/gateway/gateway/settings.py index deb1c21..e67ddd8 100644 --- a/services/gateway/gateway/settings.py +++ b/services/gateway/gateway/settings.py @@ -9,7 +9,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.2/ref/settings/ """ - +import os from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -20,10 +20,10 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-ghc+fuhbef=(q1__lbam8hivs-1it2(oo(!h$+=s*#al&p(%d*' +SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = os.getenv("DEBUG") ALLOWED_HOSTS = [ "localhost", @@ -36,7 +36,6 @@ FAUST_STORE_URL = 'rocksdb://' - # Application definition THIRD_PARTY_APPS = [ @@ -103,10 +102,10 @@ 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'farmacia-solidaria', - 'USER': 'root', - 'PASSWORD': 'root', - 'HOST': 'postgres', - 'PORT': '5432', + 'USER': os.getenv("DATABASE_USERNAME"), + 'PASSWORD': os.getenv("DATABASE_PASSWORD"), + 'HOST': os.getenv("DATABASE_HOST"), + 'PORT': os.getenv("DATABASE_PORT"), } } @@ -133,7 +132,7 @@ # Internationalization # https://docs.djangoproject.com/en/3.2/topics/i18n/ -LANGUAGE_CODE = 'pt-br' +LANGUAGE_CODE = 'en' TIME_ZONE = 'UTC' diff --git a/services/users/modules/api/actions.py b/services/users/modules/api/actions.py index 3e927da..e69795d 100644 --- a/services/users/modules/api/actions.py +++ b/services/users/modules/api/actions.py @@ -9,7 +9,7 @@ actioneer = ActionHandler() -@actioneer.register +@actioneer.register() async def auth(message: Message): try: if ( @@ -18,7 +18,7 @@ async def auth(message: Message): ): raise ActionError("You need to provide username and password") - message.data["token"] = "Fake Token" + message.data = await authorization_service.auth(message.data) except ActionError as ex: handleError( @@ -27,7 +27,7 @@ async def auth(message: Message): where="login" ) -@actioneer.register +@actioneer.register() async def register(message: Message): try: if ( @@ -35,7 +35,7 @@ async def register(message: Message): is_key_null(message.data, 'email') or is_key_null(message.data, 'password') ): - raise ActionError("You need to provide username and password") + raise ActionError("You need to provide username, password, email") message.data = await authorization_service.create_user(message.data) @@ -46,6 +46,21 @@ async def register(message: Message): where="login" ) +@actioneer.register("get") +async def permissions(message: Message): + try: + if (is_key_null(message.data, 'username')): + raise ActionError("You need to provide username and password") + + message.data = await authorization_service.get_permissions(message.data) + + except ActionError as ex: + handleError( + message, + information=ex.args[0], + where="login" + ) + @actioneer.default async def default(message: Message): handleError( diff --git a/services/users/modules/api/agents.py b/services/users/modules/api/agents.py index f039377..30b9e6b 100644 --- a/services/users/modules/api/agents.py +++ b/services/users/modules/api/agents.py @@ -13,7 +13,7 @@ async def defaultAgent(messages: StreamT[Message]): if checkError(event, 'users'): yield event - action = actioneer.get_action(event.action) + action = actioneer.get_action(event.method, event.action) await action(event) yield event \ No newline at end of file diff --git a/services/users/modules/authorization/forms.py b/services/users/modules/authorization/forms.py new file mode 100644 index 0000000..7b3a68f --- /dev/null +++ b/services/users/modules/authorization/forms.py @@ -0,0 +1,19 @@ +from django import forms +from django.contrib.auth.models import User +from django.contrib.auth.forms import UserCreationForm + +class UserRegisterForm(UserCreationForm): + email = forms.EmailField() + first_name = forms.CharField(max_length=150, required=False) + last_name = forms.CharField(max_length=150, required=False) + + class Meta: + model = User + fields = [ + 'username', + 'first_name', + 'last_name', + 'email', + 'password1', + 'password2' + ] \ No newline at end of file diff --git a/services/users/modules/authorization/models.py b/services/users/modules/authorization/models.py new file mode 100644 index 0000000..ab85ade --- /dev/null +++ b/services/users/modules/authorization/models.py @@ -0,0 +1,14 @@ +from django.db import models +from django.contrib.auth.models import User + +class Role(models.Model): + + users = models.ManyToManyField(User) + name = models.CharField("name", max_length=64, primary_key=True) + + class Meta: + verbose_name = "Role" + verbose_name_plural = "Roles" + + def __str__(self): + return self.name \ No newline at end of file diff --git a/services/users/modules/authorization/services.py b/services/users/modules/authorization/services.py index 248c80e..b91887d 100644 --- a/services/users/modules/authorization/services.py +++ b/services/users/modules/authorization/services.py @@ -1,15 +1,46 @@ +import jwt + from django.contrib.auth.models import User -from django.contrib.auth.forms import UserCreationForm +from django.contrib.auth import authenticate from asgiref.sync import sync_to_async +from common.error.error import ActionError +from common.utils.information import get_private_key + +from modules.authorization.forms import UserRegisterForm + @sync_to_async def create_user(data): data['password1'] = data['password'] data['password2'] = data['password'] - form = UserCreationForm(data) + form = UserRegisterForm(data) if form.is_valid(): form.save() return True return form.errors + +@sync_to_async +def auth(data): + user = authenticate(username=data['username'], password=data['password']) + if user is not None: + payload = { + "permissions": [role.name for role in user.role_set.iterator()] + } + token = jwt.encode(payload, get_private_key(), 'RS256') + + return { + "token": token + } + raise ActionError("Login not authorized") + + +@sync_to_async +def get_permissions(data): + user = User.objects.get(username=data['username']) + if user is not None: + return { + "permissions": [role.name for role in user.role_set.iterator()] + } + raise ActionError("User not found", ) \ No newline at end of file diff --git a/services/users/setup.py b/services/users/setup.py index 84a5a4b..7486605 100644 --- a/services/users/setup.py +++ b/services/users/setup.py @@ -13,6 +13,10 @@ "faust[rocksdb]==1.10.4", "kafka-python==1.4.7", + #Others + "PyJWT==2.1.0", + "cryptography==3.4.8", + #Build "build==0.6.0.post1", ] diff --git a/services/users/users/settings.py b/services/users/users/settings.py index 73c956a..8a28c1e 100644 --- a/services/users/users/settings.py +++ b/services/users/users/settings.py @@ -3,7 +3,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent -SECRET_KEY = os.getenv("SECRET_KEY") +SECRET_KEY = os.getenv("DJANGO_SECRET_KEY") DEBUG = os.getenv("DEBUG") diff --git a/services/users/users/urls.py b/services/users/users/urls.py index ef05f1f..52078b3 100644 --- a/services/users/users/urls.py +++ b/services/users/users/urls.py @@ -3,6 +3,6 @@ from django.urls import path, include urlpatterns = [ - # url(r'^admin/', admin.site.urls), + url('admin/', admin.site.urls), # path("api/", include('modules.api.urls')) ] From c347edfc7f979a4de678cc5480be8017d483ad36 Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Fri, 24 Sep 2021 22:30:53 -0300 Subject: [PATCH 4/7] Updated secrets --- secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secrets b/secrets index b1e8edc..250551b 160000 --- a/secrets +++ b/secrets @@ -1 +1 @@ -Subproject commit b1e8edc1d35eb494f11e2079ef14f6262852d6ed +Subproject commit 250551be4dc5945fbcd04af6b726867ac637104c From d86342af62a01a7e2c309dfb8295eed0f73ee837 Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Wed, 29 Sep 2021 22:45:29 -0300 Subject: [PATCH 5/7] Added guard and permissions system #8 --- common/error/error.py | 12 +++++ common/kafka/guard.py | 32 +++++++++++ common/kafka/send.py | 8 +-- common/models/message.py | 3 +- common/shortcuts/__init__.py | 0 common/shortcuts/classes.py | 39 ++++++++++++++ common/utils/auth.py | 18 +++++++ docker-compose.yml | 3 ++ secrets | 2 +- services/gateway/connections/users/urls.py | 4 +- services/gateway/connections/users/views.py | 53 +------------------ services/gateway/requirements.txt | 6 ++- services/users/modules/api/actions.py | 45 ++++++++++++---- .../users/modules/authorization/services.py | 3 ++ 14 files changed, 159 insertions(+), 69 deletions(-) create mode 100644 common/kafka/guard.py create mode 100644 common/shortcuts/__init__.py create mode 100644 common/shortcuts/classes.py create mode 100644 common/utils/auth.py diff --git a/common/error/error.py b/common/error/error.py index 4793e6d..86c0e0e 100644 --- a/common/error/error.py +++ b/common/error/error.py @@ -1,2 +1,14 @@ +import os class ActionError(Exception): + + def __init__(self, information, status=400, where="undefined") -> None: + self.information = information + self.status = status + self.where = self.where + + if where == "undefined" and 'NAME' in os.environ: + self.where = os.environ['NAME'] + + super().__init__(self.information) + pass \ No newline at end of file diff --git a/common/kafka/guard.py b/common/kafka/guard.py new file mode 100644 index 0000000..912356b --- /dev/null +++ b/common/kafka/guard.py @@ -0,0 +1,32 @@ +import functools + +import jwt + +from common.utils.auth import get_token_permissions +from common.models.message import Message +from common.error.error import ActionError + +def permissions_needed(permissions_needed: 'list[str]') -> 'function': + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + + permissions = [] + + if type(args[0]) is Message: + message: Message = args[0] + permissions = get_token_permissions(message.token) + + if set(permissions_needed).issubset(permissions) or 'admin' in permissions: + return func(*args, **kwargs) + + raise ActionError( + information="Permission denied", + status=401 + ) + + + return wrapper + + return decorator diff --git a/common/kafka/send.py b/common/kafka/send.py index 85f11f6..0e207d2 100644 --- a/common/kafka/send.py +++ b/common/kafka/send.py @@ -28,7 +28,7 @@ async def async_send_and_wait_message(service, action, data, filter=False, suppr return value -def send_and_wait_message(method, service, action, data, filter=False, suppress_errors=False) -> dict: +def send_and_wait_message(method, service, action, data, token, filter=False, suppress_errors=False) -> dict: key = str(uuid.uuid1()) consumer = build_kafka_consumer(service+"-outcome", timeout_in_seconds=10) @@ -39,7 +39,8 @@ def send_and_wait_message(method, service, action, data, filter=False, suppress_ service=service, action=action, data=data, - method=method, + method=method, + token=token, key=key ) @@ -51,13 +52,14 @@ def send_and_wait_message(method, service, action, data, filter=False, suppress_ value = _filter_value(value) return value -def send_message(service, action, data, method, key=""): +def send_message(service, action, data, method, token, key=""): finalKey = key if key != "" else str(uuid.uuid1()) message = Message( action=action, method=method, data=data, + token=token, id=finalKey, ) try: diff --git a/common/models/message.py b/common/models/message.py index df3f978..81926b2 100644 --- a/common/models/message.py +++ b/common/models/message.py @@ -5,4 +5,5 @@ class Message(faust.Record, serializer='json'): data: dict id: str method: str - error: bool = False \ No newline at end of file + error: bool = False + token: str = "" \ No newline at end of file diff --git a/common/shortcuts/__init__.py b/common/shortcuts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/shortcuts/classes.py b/common/shortcuts/classes.py new file mode 100644 index 0000000..3057901 --- /dev/null +++ b/common/shortcuts/classes.py @@ -0,0 +1,39 @@ +from django.http.request import HttpRequest +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView +from common.kafka.send import send_and_wait_message + +class SimpleConnection(APIView): + + def __init__(self, name=None): + + self.name = name if name else self.__class__.__name__.lower() + + def _default_send_routine(self, method: str, request: HttpRequest, action: str): + + token = request.headers.get("Authorization") or "" + + data = send_and_wait_message( + service=self.name, + method=method, + action=action, + data=request.data, + filter=True, + suppress_errors=True, + token=token + ) + + if data: + return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) + + return Response(status=status.HTTP_408_REQUEST_TIMEOUT) + + + def post(self, request, action): return self._default_send_routine('post', request, action) + def get(self, request, action): return self._default_send_routine( 'get', request, action) + def put(self, request, action): return self._default_send_routine('put', request, action) + def patch(self, request, action): return self._default_send_routine('patch', request, action) + def delete(self, request, action): return self._default_send_routine('delete', request, action) + def options(self, request, action): return self._default_send_routine('options', request, action) + def head(self, request, action): return self._default_send_routine('head', request, action) \ No newline at end of file diff --git a/common/utils/auth.py b/common/utils/auth.py new file mode 100644 index 0000000..a6e054c --- /dev/null +++ b/common/utils/auth.py @@ -0,0 +1,18 @@ +from common.error.error import ActionError +import jwt + +from common.utils.information import get_public_key + +def get_token_permissions(token): + try: + data = jwt.decode(token, get_public_key(), 'RS256') + + return data['permissions'] + except jwt.ExpiredSignatureError: + raise ActionError( + information="Token has expired", + status=403 + ) + except: + return [] + diff --git a/docker-compose.yml b/docker-compose.yml index 93269d5..5b056f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - SERVICE_PORT=8000 - DATABASE_HOST=postgres - DATABASE_PORT=5432 + - NAME=gateway - DEBUG - DJANGO_SECRET_KEY - DATABASE_USERNAME @@ -43,12 +44,14 @@ services: - SERVICE_PORT=8001 - DATABASE_HOST=postgres - DATABASE_PORT=5432 + - NAME=users - DEBUG - DJANGO_SECRET_KEY - DATABASE_USERNAME - DATABASE_PASSWORD - PRIVATE_KEY - PUBLIC_KEY + - EXPIRATION_TIME depends_on: - kafka - postgres diff --git a/secrets b/secrets index 250551b..e576815 160000 --- a/secrets +++ b/secrets @@ -1 +1 @@ -Subproject commit 250551be4dc5945fbcd04af6b726867ac637104c +Subproject commit e5768155fb9e96d941f60ae2ab21ef169eefb8b7 diff --git a/services/gateway/connections/users/urls.py b/services/gateway/connections/users/urls.py index 1e9e8e6..db1cabf 100644 --- a/services/gateway/connections/users/urls.py +++ b/services/gateway/connections/users/urls.py @@ -1,8 +1,8 @@ from django.contrib import admin from django.urls import path -from .views import UserViewset +from .views import UserConnection urlpatterns = [ - path('users/', UserViewset.as_view()), + path('users/', UserConnection.as_view()), ] diff --git a/services/gateway/connections/users/views.py b/services/gateway/connections/users/views.py index 62dd530..6bbdaf8 100644 --- a/services/gateway/connections/users/views.py +++ b/services/gateway/connections/users/views.py @@ -1,52 +1,3 @@ -from django.http.response import JsonResponse -from rest_framework import viewsets, status -from rest_framework.response import Response -from rest_framework.views import APIView -from common.kafka.send import send_and_wait_message +from common.shortcuts.classes import SimpleConnection -class UserViewset(APIView): - - def post(self, request, action): - data = send_and_wait_message( - service="users", - method="post", - action=action, - data=request.data, - filter=True, - suppress_errors=True - ) - - if data: - return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) - - return Response(status=status.HTTP_408_REQUEST_TIMEOUT) - - def get(self, request, action): - data = send_and_wait_message( - service="users", - method="get", - action=action, - data=request.data, - filter=True, - suppress_errors=True - ) - - if data: - return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) - - return Response(status=status.HTTP_408_REQUEST_TIMEOUT) - - def delete(self, request, action): - data = send_and_wait_message( - service="users", - method="delete", - action=action, - data=request.data, - filter=True, - suppress_errors=True - ) - - if data: - return Response(data, status=data["data"]["status"] if data["error"] else status.HTTP_200_OK) - - return Response(status=status.HTTP_408_REQUEST_TIMEOUT) \ No newline at end of file +class UserConnection(SimpleConnection): pass \ No newline at end of file diff --git a/services/gateway/requirements.txt b/services/gateway/requirements.txt index 0c1bf10..0ed72f4 100644 --- a/services/gateway/requirements.txt +++ b/services/gateway/requirements.txt @@ -9,4 +9,8 @@ aiohttp==3.7.4 # Kafka Libs faust[rocksdb]==1.10.4 eventlet==0.31.1 -kafka-python==1.4.7 \ No newline at end of file +kafka-python==1.4.7 + +# Others +PyJWT==2.1.0 +cryptography==3.4.8 \ No newline at end of file diff --git a/services/users/modules/api/actions.py b/services/users/modules/api/actions.py index e69795d..a84b790 100644 --- a/services/users/modules/api/actions.py +++ b/services/users/modules/api/actions.py @@ -3,11 +3,31 @@ from common.kafka.actions import ActionHandler from common.error.handling import handleError from common.utils.validation import is_key_null +from common.kafka.guard import permissions_needed import modules.authorization.services as authorization_service actioneer = ActionHandler() +@actioneer.register() +@permissions_needed('namorada') +async def get_self_permissions(message: Message): + try: + if ( + is_key_null(message.data, 'username') or + is_key_null(message.data, 'password') + ): + raise ActionError(information="You need to provide username and password") + + message.data = await authorization_service.auth(message.data) + + except ActionError as ex: + handleError( + message, + information=ex.information, + status=ex.status, + where=ex.where + ) @actioneer.register() async def auth(message: Message): @@ -16,15 +36,16 @@ async def auth(message: Message): is_key_null(message.data, 'username') or is_key_null(message.data, 'password') ): - raise ActionError("You need to provide username and password") + raise ActionError(information="You need to provide username and password") message.data = await authorization_service.auth(message.data) except ActionError as ex: handleError( message, - information=ex.args[0], - where="login" + information=ex.information, + status=ex.status, + where=ex.where ) @actioneer.register() @@ -35,30 +56,32 @@ async def register(message: Message): is_key_null(message.data, 'email') or is_key_null(message.data, 'password') ): - raise ActionError("You need to provide username, password, email") + raise ActionError(information="You need to provide username, password, email") message.data = await authorization_service.create_user(message.data) except ActionError as ex: handleError( message, - information=ex.args[0], - where="login" + information=ex.information, + status=ex.status, + where=ex.where ) @actioneer.register("get") async def permissions(message: Message): try: if (is_key_null(message.data, 'username')): - raise ActionError("You need to provide username and password") + raise ActionError(information="You need to provide username and password") message.data = await authorization_service.get_permissions(message.data) except ActionError as ex: handleError( message, - information=ex.args[0], - where="login" + information=ex.information, + status=ex.status, + where=ex.where ) @actioneer.default @@ -66,4 +89,6 @@ async def default(message: Message): handleError( message, information="Action not implemented", - where="users") + where="users", + status=404 + ) diff --git a/services/users/modules/authorization/services.py b/services/users/modules/authorization/services.py index b91887d..54df664 100644 --- a/services/users/modules/authorization/services.py +++ b/services/users/modules/authorization/services.py @@ -1,4 +1,6 @@ +import os import jwt +import datetime from django.contrib.auth.models import User from django.contrib.auth import authenticate @@ -26,6 +28,7 @@ def auth(data): user = authenticate(username=data['username'], password=data['password']) if user is not None: payload = { + "exp": datetime.datetime.utcnow() + datetime.timedelta(seconds=60*int(os.environ["EXPIRATION_TIME"])), "permissions": [role.name for role in user.role_set.iterator()] } token = jwt.encode(payload, get_private_key(), 'RS256') From 9c8d793b107900c17b6d4dd82de8faff1b4386a2 Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Thu, 30 Sep 2021 18:10:21 -0300 Subject: [PATCH 6/7] Fixed auth system and finished guard #8 --- common/error/error.py | 2 +- common/kafka/guard.py | 5 +- common/shortcuts/classes.py | 5 +- common/utils/functions.py | 2 + services/gateway/connections/users/urls.py | 4 +- services/gateway/connections/users/views.py | 2 +- services/users/modules/api/actions.py | 95 +++++++-------------- services/users/modules/api/agents.py | 26 ++++-- 8 files changed, 60 insertions(+), 81 deletions(-) create mode 100644 common/utils/functions.py diff --git a/common/error/error.py b/common/error/error.py index 86c0e0e..4834519 100644 --- a/common/error/error.py +++ b/common/error/error.py @@ -4,7 +4,7 @@ class ActionError(Exception): def __init__(self, information, status=400, where="undefined") -> None: self.information = information self.status = status - self.where = self.where + self.where = where if where == "undefined" and 'NAME' in os.environ: self.where = os.environ['NAME'] diff --git a/common/kafka/guard.py b/common/kafka/guard.py index 912356b..31eadf0 100644 --- a/common/kafka/guard.py +++ b/common/kafka/guard.py @@ -3,6 +3,7 @@ import jwt from common.utils.auth import get_token_permissions +from common.utils.functions import treat_token from common.models.message import Message from common.error.error import ActionError @@ -16,9 +17,9 @@ def wrapper(*args, **kwargs): if type(args[0]) is Message: message: Message = args[0] - permissions = get_token_permissions(message.token) + permissions = get_token_permissions(treat_token(message.token)) - if set(permissions_needed).issubset(permissions) or 'admin' in permissions: + if len(set(permissions_needed).intersection(permissions)) > 0 or 'admin' in permissions: return func(*args, **kwargs) raise ActionError( diff --git a/common/shortcuts/classes.py b/common/shortcuts/classes.py index 3057901..1207987 100644 --- a/common/shortcuts/classes.py +++ b/common/shortcuts/classes.py @@ -7,8 +7,9 @@ class SimpleConnection(APIView): def __init__(self, name=None): - - self.name = name if name else self.__class__.__name__.lower() + className = self.__class__.__name__.lower().split("connection")[0] + self.name = className if name is None else name + def _default_send_routine(self, method: str, request: HttpRequest, action: str): diff --git a/common/utils/functions.py b/common/utils/functions.py new file mode 100644 index 0000000..c24fbce --- /dev/null +++ b/common/utils/functions.py @@ -0,0 +1,2 @@ +def treat_token(token): + return token.split('Bearer ')[1] \ No newline at end of file diff --git a/services/gateway/connections/users/urls.py b/services/gateway/connections/users/urls.py index db1cabf..3fc7341 100644 --- a/services/gateway/connections/users/urls.py +++ b/services/gateway/connections/users/urls.py @@ -1,8 +1,8 @@ from django.contrib import admin from django.urls import path -from .views import UserConnection +from .views import UsersConnection urlpatterns = [ - path('users/', UserConnection.as_view()), + path('users/', UsersConnection.as_view()), ] diff --git a/services/gateway/connections/users/views.py b/services/gateway/connections/users/views.py index 6bbdaf8..67b41a1 100644 --- a/services/gateway/connections/users/views.py +++ b/services/gateway/connections/users/views.py @@ -1,3 +1,3 @@ from common.shortcuts.classes import SimpleConnection -class UserConnection(SimpleConnection): pass \ No newline at end of file +class UsersConnection(SimpleConnection): pass \ No newline at end of file diff --git a/services/users/modules/api/actions.py b/services/users/modules/api/actions.py index a84b790..eb5aa21 100644 --- a/services/users/modules/api/actions.py +++ b/services/users/modules/api/actions.py @@ -1,94 +1,57 @@ from common.error.error import ActionError from common.models.message import Message from common.kafka.actions import ActionHandler -from common.error.handling import handleError from common.utils.validation import is_key_null from common.kafka.guard import permissions_needed +from common.utils.functions import treat_token +from common.utils.auth import get_token_permissions import modules.authorization.services as authorization_service actioneer = ActionHandler() @actioneer.register() -@permissions_needed('namorada') +@permissions_needed(['atendente', 'gerente']) async def get_self_permissions(message: Message): - try: - if ( - is_key_null(message.data, 'username') or - is_key_null(message.data, 'password') - ): - raise ActionError(information="You need to provide username and password") - - message.data = await authorization_service.auth(message.data) - - except ActionError as ex: - handleError( - message, - information=ex.information, - status=ex.status, - where=ex.where - ) + message.data = get_token_permissions(treat_token(message.token)) -@actioneer.register() -async def auth(message: Message): - try: - if ( - is_key_null(message.data, 'username') or - is_key_null(message.data, 'password') - ): - raise ActionError(information="You need to provide username and password") - message.data = await authorization_service.auth(message.data) - - except ActionError as ex: - handleError( - message, - information=ex.information, - status=ex.status, - where=ex.where - ) +@actioneer.register('get') +async def auth(message: Message): + if ( + is_key_null(message.data, 'username') or + is_key_null(message.data, 'password') + ): + raise ActionError(information="You need to provide username and password") + + message.data = await authorization_service.auth(message.data) + + @actioneer.register() async def register(message: Message): - try: - if ( - is_key_null(message.data, 'username') or - is_key_null(message.data, 'email') or - is_key_null(message.data, 'password') - ): - raise ActionError(information="You need to provide username, password, email") - - message.data = await authorization_service.create_user(message.data) + if ( + is_key_null(message.data, 'username') or + is_key_null(message.data, 'email') or + is_key_null(message.data, 'password') + ): + raise ActionError(information="You need to provide username, password, email") + + message.data = await authorization_service.create_user(message.data) + - except ActionError as ex: - handleError( - message, - information=ex.information, - status=ex.status, - where=ex.where - ) @actioneer.register("get") async def permissions(message: Message): - try: - if (is_key_null(message.data, 'username')): - raise ActionError(information="You need to provide username and password") - - message.data = await authorization_service.get_permissions(message.data) + if (is_key_null(message.data, 'username')): + raise ActionError(information="You need to provide username and password") + + message.data = await authorization_service.get_permissions(message.data) - except ActionError as ex: - handleError( - message, - information=ex.information, - status=ex.status, - where=ex.where - ) @actioneer.default async def default(message: Message): - handleError( - message, + raise ActionError( information="Action not implemented", - where="users", status=404 ) diff --git a/services/users/modules/api/agents.py b/services/users/modules/api/agents.py index 30b9e6b..0adbd01 100644 --- a/services/users/modules/api/agents.py +++ b/services/users/modules/api/agents.py @@ -1,6 +1,7 @@ +from common.error.error import ActionError from faust.types import StreamT -from common.error.handling import checkError +from common.error.handling import handleError, checkError from common.models.message import Message from modules.api.actions import actioneer @@ -10,10 +11,21 @@ @faustApp.agent(defaultInTopic, sink=[defaultOutTopic]) async def defaultAgent(messages: StreamT[Message]): async for event in messages: - if checkError(event, 'users'): + try: + if checkError(event, 'users'): + yield event + + action = actioneer.get_action(event.method, event.action) + await action(event) + + except ActionError as ex: + handleError( + event, + information=ex.information, + status=ex.status, + where=ex.where + ) + + finally: yield event - - action = actioneer.get_action(event.method, event.action) - await action(event) - - yield event \ No newline at end of file + \ No newline at end of file From 49f09a69ec474baf4dfdd1bf24e3e9e14b7a2d5f Mon Sep 17 00:00:00 2001 From: Otavio Henrique Date: Thu, 30 Sep 2021 19:19:53 -0300 Subject: [PATCH 7/7] Fixed typo #8 --- services/users/modules/api/actions.py | 8 +++----- services/users/modules/api/agents.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/services/users/modules/api/actions.py b/services/users/modules/api/actions.py index eb5aa21..8475ad5 100644 --- a/services/users/modules/api/actions.py +++ b/services/users/modules/api/actions.py @@ -10,12 +10,11 @@ actioneer = ActionHandler() -@actioneer.register() +@actioneer.register('get') @permissions_needed(['atendente', 'gerente']) -async def get_self_permissions(message: Message): +async def self_permissions(message: Message): message.data = get_token_permissions(treat_token(message.token)) - @actioneer.register('get') async def auth(message: Message): @@ -26,8 +25,8 @@ async def auth(message: Message): raise ActionError(information="You need to provide username and password") message.data = await authorization_service.auth(message.data) - + @actioneer.register() async def register(message: Message): if ( @@ -40,7 +39,6 @@ async def register(message: Message): message.data = await authorization_service.create_user(message.data) - @actioneer.register("get") async def permissions(message: Message): if (is_key_null(message.data, 'username')): diff --git a/services/users/modules/api/agents.py b/services/users/modules/api/agents.py index 0adbd01..99f727f 100644 --- a/services/users/modules/api/agents.py +++ b/services/users/modules/api/agents.py @@ -8,7 +8,7 @@ from modules.api.topics import defaultInTopic, defaultOutTopic from modules.kafka_handler.app import faustApp -@faustApp.agent(defaultInTopic, sink=[defaultOutTopic]) +@faustApp.agent(defaultInTopic, sink=[defaultOutTopic], concurrency=10) async def defaultAgent(messages: StreamT[Message]): async for event in messages: try: