diff --git a/backend/gn_module_export/blueprint.py b/backend/gn_module_export/blueprint.py index 9b8a4272..d8835a6d 100644 --- a/backend/gn_module_export/blueprint.py +++ b/backend/gn_module_export/blueprint.py @@ -29,7 +29,7 @@ from flask_admin.contrib.sqla import ModelView from flask_admin.helpers import is_form_submitted from flask_admin.babel import gettext -from werkzeug.exceptions import NotFound, BadRequest +from werkzeug.exceptions import NotFound, BadRequest, Forbidden from wtforms import validators from pypnusershub.db.models import User @@ -46,10 +46,10 @@ import gn_module_export.tasks # noqua: F401 -from .repositories import ExportObjectQueryRepository, generate_swagger_spec +from .repositories import generate_swagger_spec from .models import Export, CorExportsRoles, Licences, ExportSchedules, UserRepr -from .utils_export import thread_export_data from .commands import commands +from .tasks import generate_export LOGGER = current_app.logger LOGGER.setLevel(logging.DEBUG) @@ -374,54 +374,43 @@ def getOneExportThread(id_export, export_format): """ Run export with thread """ - # Test if export exists - if ( - id_export < 1 - or export_format not in current_app.config["EXPORTS"]["export_format_map"] - ): + + filters = {f: request.args.get(f) for f in request.args} + data = dict(request.get_json()) + user = g.current_user + + # Test format + if export_format not in current_app.config["EXPORTS"]["export_format_map"]: return to_json_resp( { "api_error": "invalid_export", - "message": "Invalid export or export not found", + "message": "Invalid export format", }, - status=404, + status=500, ) - filters = {f: request.args.get(f) for f in request.args} - data = dict(request.get_json()) + export = Export.query.get(id_export) + if not export: + return jsonify([]) + if not export.has_instance_permission(user.id_role): + raise Forbidden + + # Test email # Alternative email in payload email_to = None if "email" in data: email_to = data["email"] - - @copy_current_request_context - def get_data(id_export, export_format, role, filters, email_to): - thread_export_data(id_export, export_format, role, filters, email_to) - - exp = ExportObjectQueryRepository(id_export=id_export, role=g.current_user) - # Test if user have an email - try: - user = g.current_user - if not user.email and not email_to: # TODO add more test - raise BadRequest("User doesn't have email") - except NoResultFound: - raise NotFound("User doesn't exist") - - # Run export - a = threading.Thread( - name="export_data", - target=get_data, - kwargs={ - "id_export": id_export, - "export_format": export_format, - "role": g.current_user, - "filters": filters, - "email_to": [email_to] if (email_to) else [user.email], - }, + if not user.email and not email_to: # TODO add more test + raise BadRequest("User doesn't have email") + + generate_export.delay( + export_id=id_export, + export_format=export_format, + scheduled=False, + skip_newer_than=None, ) - a.start() return to_json_resp( { @@ -509,9 +498,20 @@ def get_one_export_api(id_export): order by : @TODO """ + user = g.current_user + export = Export.query.get(id_export) + + if not export: + return jsonify([]) + if not export.has_instance_permission(user.id_role): + raise Forbidden + limit = request.args.get("limit", default=1000, type=int) offset = request.args.get("offset", default=0, type=int) + if limit > 1000: + limit = 1000 + args = request.args.to_dict() if "limit" in args: args.pop("limit") @@ -519,14 +519,15 @@ def get_one_export_api(id_export): args.pop("offset") filters = {f: args.get(f) for f in args} - exprep = ExportObjectQueryRepository( - id_export=id_export, - role=g.current_user, - filters=filters, - limit=limit, - offset=offset, - ) - data = exprep.get_export_with_logging() + query = export.get_view_query(limit=limit, offset=offset, filters=filters) + + data = query.return_query() + + export_license = (export.as_dict(fields=["licence"])).get("licence", None) + data["license"] = dict() + data["license"]["name"] = export_license.get("name_licence", None) + data["license"]["href"] = export_license.get("url_licence", None) + return data diff --git a/backend/gn_module_export/models.py b/backend/gn_module_export/models.py index 0836d327..b9842576 100644 --- a/backend/gn_module_export/models.py +++ b/backend/gn_module_export/models.py @@ -13,6 +13,8 @@ from geonature.utils.env import DB from utils_flask_sqla.serializers import serializable +from utils_flask_sqla_geo.generic import GenericQueryGeo + from pypnusershub.db.models import User from geonature.core.users.models import CorRole @@ -82,7 +84,7 @@ class Export(DB.Model): id_licence = DB.Column( DB.Integer(), DB.ForeignKey(Licences.id_licence), nullable=False ) - allowed_roles = DB.relationship("CorExportsRoles") + licence = DB.relationship("Licences") def __str__(self): @@ -90,6 +92,47 @@ def __str__(self): __repr__ = __str__ + def has_instance_permission(self, id_role=None): + if self.public: + return True + + user = User.query.get(id_role) + if not user: + return False + if user in self.allowed_roles: + return True + + return False + + def get_view_query(self, limit, offset, filters=None): + return GenericQueryGeo( + DB, + self.view_name, + self.schema_name, + filters, + limit, + offset, + self.geometry_field, + ) + + +class CorExportsRoles(DB.Model): + __tablename__ = "cor_exports_roles" + __table_args__ = {"schema": "gn_exports"} + id_export = DB.Column( + DB.Integer, DB.ForeignKey(Export.id), primary_key=True, nullable=False + ) + + id_role = DB.Column( + DB.Integer, DB.ForeignKey(User.id_role), primary_key=True, nullable=False + ) + + # export = DB.relationship("Export", cascade="all,delete") + # role = DB.relationship("UserRepr") + + +Export.allowed_roles = DB.relationship(User, secondary=CorExportsRoles.__table__) + @serializable class ExportLog(DB.Model): @@ -117,21 +160,6 @@ def record(cls, adict): DB.session.rollback() -class CorExportsRoles(DB.Model): - __tablename__ = "cor_exports_roles" - __table_args__ = {"schema": "gn_exports"} - id_export = DB.Column( - DB.Integer(), DB.ForeignKey(Export.id), primary_key=True, nullable=False - ) - - id_role = DB.Column( - DB.Integer, DB.ForeignKey(User.id_role), primary_key=True, nullable=False - ) - - export = DB.relationship("Export", lazy="joined", cascade="all,delete") - role = DB.relationship("UserRepr", lazy="joined") - - class ExportSchedules(DB.Model): __tablename__ = "t_export_schedules" __table_args__ = {"schema": "gn_exports"} diff --git a/backend/gn_module_export/repositories.py b/backend/gn_module_export/repositories.py index 804879e8..31de9e19 100644 --- a/backend/gn_module_export/repositories.py +++ b/backend/gn_module_export/repositories.py @@ -2,179 +2,14 @@ Module de gestion des exports """ -import sys -import logging - -from werkzeug.exceptions import Forbidden -from datetime import datetime -from sqlalchemy.orm.exc import NoResultFound - -from flask import current_app, g - - from geonature.utils.env import DB -# from utils_flask_sqla.generic import GenericQuery, GenericTable -from utils_flask_sqla_geo.generic import GenericQueryGeo, GenericTableGeo +from utils_flask_sqla_geo.generic import GenericTableGeo from .models import Export, ExportLog, ExportSchedules -class ExportObjectQueryRepository: - def __init__(self, id_export, role=None, filters=None, limit=1000, offset=0): - """ - Classe permettant de manipuler l'objet export - Permet d'interroger la vue définie par l'export - et de récupérer les données - - Args: - id_export (int): Indentifiant de l'export (t_exports) - session ([type], optional): [description]. Defaults to DB.session. - filters ({}, optional):Filtres à appliquer sur les données. Defaults to None. - limit (int, optional): Nombre maximum de données à retourner. Defaults to 1000. - offset (int, optional): Numéro de page à retourner. Defaults to 0. - """ - # Test si l'export est autorisé - self.id_export = id_export - self.role = role - try: - self.export = Export.query.get(self.id_export) - if self.role: - self.export = self.get_export_is_allowed() - except NoResultFound: - raise Forbidden( - ('Not allowed to access to export : "{}"').format( - self.id_export, - ) - ) - - if not filters: - filters = dict() - - self.exportobject_query_definition = GenericQueryGeo( - DB, - self.export.view_name, - self.export.schema_name, - filters, - limit, - offset, - self.export.geometry_field, - ) - - def _get_export_columns_definition(self): - """ - Export de la définition des colonnes de la vue - - """ - return self.exportobject_query_definition.view.db_cols - - def _get_data(self, format="csv"): - """ - Fonction qui retourne les données de l'export passé en paramètre - en appliquant des filtres s'il y a lieu - - .. :quickref: lance une requete qui récupère les données - pour un export donné - - - **Returns:** - - .. sourcecode:: http - - { - 'total': Number total of results, - 'total_filtered': Number of results after filteer , - 'page': Page number, - 'limit': Limit, - 'items': data on GeoJson format - 'licence': information of licence associated to data - } - - """ - - # Export dilimitfférent selon le format demandé - # shp ou geojson => geo_feature - # json => - - EXPORT_FORMAT = current_app.config["EXPORTS"]["export_format_map"] - if self.export.geometry_field and EXPORT_FORMAT[format]["geofeature"]: - data = self.exportobject_query_definition.as_geofeature() - else: - data = self.exportobject_query_definition.return_query() - # Ajout licence - if self.export: - try: - export_license = (self.export.as_dict(fields=["licence"])).get( - "licence", None - ) - data["license"] = dict() - data["license"]["name"] = export_license.get("name_licence", None) - data["license"]["href"] = export_license.get("url_licence", None) - except Exception as e: - print(e) - pass - return data - - def get_export_with_logging(self, export_format="json"): - """ - Fonction qui retourne les données pour un export données - et qui enregistre l'opération dans la table des logs - - .. :quickref: retourne les données pour un export données - - - :query str export_format: format de l'export (csv, json, shp) - - **Returns:** - - .. sourcecode:: http - - { - 'total': Number total of results, - 'total_filtered': Number of results after filteer , - 'page': Page number, - 'limit': Limit, - 'items': data on GeoJson format - 'licence': information of licence associated to data - } - - - """ - log = None - status = -2 - start_time = datetime.utcnow() - end_time = None - - data = self._get_data(format=export_format) - - status = 0 - - end_time = datetime.utcnow() - - ExportLog.record( - { - "id_role": self.role.id_role, - "id_export": self.export.id, - "format": export_format, - "start_time": start_time, - "end_time": end_time, - "status": status, - "log": log, - } - ) - return data - - def get_export_is_allowed(self): - """ - Test si un role a les droits sur un export - """ - q = Export.query.filter(Export.id == self.id_export).get_allowed_exports( - user=self.role - ) - return q.one() - - SWAGGER_TYPE_COR = { "INTEGER": {"type": "int", "format": "int32"}, "BIGINT": {"type": "int", "format": "int64"}, diff --git a/backend/gn_module_export/utils_export.py b/backend/gn_module_export/utils_export.py index 06f616cb..334cce72 100644 --- a/backend/gn_module_export/utils_export.py +++ b/backend/gn_module_export/utils_export.py @@ -17,8 +17,8 @@ from utils_flask_sqla_geo.utilsgeometry import FionaShapeService, FionaGpkgService from geonature.utils.filemanager import removeDisallowedFilenameChars -from .repositories import ExportObjectQueryRepository from .send_mail import export_send_mail, export_send_mail_error +from .models import Export class ExportGenerationNotNeeded(Exception): @@ -42,69 +42,6 @@ def schedule_export_filename(export): return "{}".format(removeDisallowedFilenameChars(export.get("label"))) -def thread_export_data(id_export, export_format, role, filters, mail_to): - """ - Lance un thread qui permet d'exécuter les fonctions d'export - en arrière plan - - .. :quickref: Lance un thread qui permet d'exécuter les fonctions - d'export en arrière plan - - :query int id_export: Identifiant de l'export - :query str export_format: Format de l'export (csv, json, shp) - :query {} role: Role - :query {} filters: Filtre à appliquer sur l'export - :query [str] mail_to: Email de reception - - - **Returns:** - .. void - """ - - exprep = ExportObjectQueryRepository( - id_export=id_export, role=role, filters=filters, limit=-1, offset=0 - ) - - # export data - try: - data = exprep.get_export_with_logging(export_format=export_format) - columns = exprep._get_export_columns_definition() - export_dict = exprep.export.as_dict(fields=["licence"]) - except Exception as exp: - export_send_mail_error( - mail_to, None, "Error when exporting data : {}".format(repr(exp)) - ) - return - - # Generate and store export file - try: - file_name = export_filename(export_dict) - full_file_name = GenerateExport( - file_name=file_name, - format=export_format, - data=data, - columns=columns, - export=export_dict, - ).generate_data_export() - - except Exception as exp: - export_send_mail_error( - mail_to, - export_dict, - "Error when creating the export file : {}".format(repr(exp)), - ) - raise exp - return - - # Send mail - try: - export_send_mail(mail_to=mail_to, export=export_dict, file_name=full_file_name) - except Exception as exp: - export_send_mail_error( - mail_to, export_dict, "Error when sending email : {}".format(repr(exp)) - ) - - def export_data_file( id_export, export_format, filters={}, isScheduler=False, skip_newer_than=None ): @@ -122,16 +59,26 @@ def export_data_file( .. str : nom du fichier """ - exprep = ExportObjectQueryRepository( - id_export=id_export, role=None, filters=filters, limit=-1, offset=0 - ) + export = Export.query.get(id_export) # export data - data = exprep._get_data(format=export_format) - columns = exprep._get_export_columns_definition() + query = export.get_view_query(limit=-1, offset=0, filters=filters) + + EXPORT_FORMAT = current_app.config["EXPORTS"]["export_format_map"] + if export.geometry_field and EXPORT_FORMAT[export_format]["geofeature"]: + data = query.as_geofeature() + else: + data = query.return_query() + # Ajout licence + export_license = (export.as_dict(fields=["licence"])).get("licence", None) + data["license"] = dict() + data["license"]["name"] = export_license.get("name_licence", None) + data["license"]["href"] = export_license.get("url_licence", None) + + columns = query.view.db_cols # Generate and store export file - export_def = exprep.export.as_dict() + export_def = export.as_dict() if isScheduler: file_name = schedule_export_filename(export_def) else: