diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7baf831dd..d52f6b0d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,11 @@ jobs: steps: - name: Check out iris uses: actions/checkout@v4 + - name: Check code with ruff + uses: astral-sh/ruff-action@v2 + with: + args: check --output-format=github + src: ./source - name: Build dockers run: | # TODO using the environment file from tests to build here. diff --git a/e2e/tests/administrator/alerts.spec.js b/e2e/tests/administrator/alerts.spec.js index a66eb966c..bd41e68b8 100644 --- a/e2e/tests/administrator/alerts.spec.js +++ b/e2e/tests/administrator/alerts.spec.js @@ -17,5 +17,6 @@ test('should present the alert', async ({ page, rest }) => { alert_customer_id: 1 } }); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); await expect(page.getByRole('heading', { name: alertTitle })).toBeVisible(); }); \ No newline at end of file diff --git a/e2e/tests/administrator/case/ioc.spec.js b/e2e/tests/administrator/case/ioc.spec.js index 04bee5d25..7d58a8a37 100644 --- a/e2e/tests/administrator/case/ioc.spec.js +++ b/e2e/tests/administrator/case/ioc.spec.js @@ -1,11 +1,14 @@ import { test } from '../../restFixture.js'; import { expect } from '@playwright/test'; +import Api from '../../api.js'; import crypto from 'node:crypto'; test.beforeEach(async({ page }) => { await page.goto('/case/ioc?cid=1'); }); +// TODO should maybe remove all iocs between each tests: there is a risk we reach the pagination limit + test('should be able to update IOC', async ({ page }) => { const iocValue = `IOC value - ${crypto.randomUUID()}`; @@ -43,18 +46,7 @@ test('should not be able to create an IOC with the same type and value', async ( }); test('should paginate the IOCs', async ({ page, rest }) => { - const caseName = `Case - ${crypto.randomUUID()}`; - - // TODO maybe should remove cases between each tests (like in the backend tests) - let response = await rest.post('/api/v2/cases', { - data: { - case_name: caseName, - case_description: 'Case description', - case_customer: 1, - case_soc_id: '' - } - }); - const caseIdentifier = (await response.json()).case_id; + const caseIdentifier = await Api.createCase(rest); for (let i = 0; i < 11; i++) { await rest.post(`/api/v2/cases/${caseIdentifier}/iocs`, { data: { diff --git a/e2e/tests/administrator/dim/tasks.spec.js b/e2e/tests/administrator/dim/tasks.spec.js new file mode 100644 index 000000000..b0d4346a1 --- /dev/null +++ b/e2e/tests/administrator/dim/tasks.spec.js @@ -0,0 +1,39 @@ +import { test } from '../../restFixture.js'; +import { expect } from '@playwright/test'; +import Api from '../../api.js'; +import crypto from 'node:crypto'; + +let api; + +test.beforeEach(async ({ page, rest }) => { + await page.goto('/dim/tasks'); +}); + +test('should be able to consult task info', async ({ page, rest, browserName }) => { + let response = await rest.get('/manage/modules/list'); + const modules = (await response.json()).data; + const irisCheckModule = modules.find(module => module.module_human_name === 'IrisCheck'); + response = await rest.post(`/manage/modules/enable/${irisCheckModule.id}`); + const caseIdentifier = await Api.createCase(rest); + await rest.delete(`/api/v2/cases/${caseIdentifier}`); + + await page.goto('/dim/tasks'); + // filter 'Case' column with the case identifier + // TODO should make the more interface testable to be able to use some page.getByRole + await page.locator('th:nth-child(4) > .form-group > .form-control').fill(caseIdentifier.toString()); + // filter 'Processing module' column with on_postload_case_delete + // TODO should make the more interface testable to be able to use some page.getByRole + await page.locator('th:nth-child(5) > .form-group > .form-control').fill('on_postload_case_delete'); + // TODO should make the more interface testable to be able to use some page.getByRole + await page.locator('td').getByRole('link').click(); + if (browserName === 'chromium') { + // TODO this click should not be necessary. However, when run on the chromium browser, it seem the first click is ignored + // => there is probably a bug in the code to chase + await page.locator('td').getByRole('link').click(); + } + + await expect(page.locator('#info_dim_task_modal_body')).toContainText('Module name: iris_check_module'); + await expect(page.locator('#info_dim_task_modal_body')).toContainText('Hook name: on_postload_case_delete'); + await expect(page.locator('#info_dim_task_modal_body')).toContainText('User: administrator'); + await expect(page.locator('#info_dim_task_modal_body')).toContainText(`Case ID: ${caseIdentifier}`); +}); diff --git a/e2e/tests/api.js b/e2e/tests/api.js new file mode 100644 index 000000000..b536d038d --- /dev/null +++ b/e2e/tests/api.js @@ -0,0 +1,20 @@ +import crypto from 'node:crypto'; + +const createCase = async (rest) => { + const caseName = `Case - ${crypto.randomUUID()}`; + + // TODO maybe should remove cases between each tests (like in the backend tests) + const response = await rest.post('/api/v2/cases', { + data: { + case_name: caseName, + case_description: 'Case description', + case_customer: 1, + case_soc_id: '' + } + }); + return (await response.json()).case_id; +}; + +export default { + createCase +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..101c01ea8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.ruff.lint] +ignore = ["E402", "E711", "E712", "E721", "E722", "F821"] diff --git a/source/app/__init__.py b/source/app/__init__.py index bf63eb441..64353b1df 100644 --- a/source/app/__init__.py +++ b/source/app/__init__.py @@ -24,11 +24,11 @@ from flask import session from flask_bcrypt import Bcrypt from flask_caching import Cache -from flask_cors import CORS from flask_login import LoginManager from flask_marshmallow import Marshmallow -from flask_socketio import SocketIO, Namespace +from flask_socketio import SocketIO +from flask_socketio import Namespace from flask_sqlalchemy import SQLAlchemy from functools import partial @@ -158,4 +158,21 @@ def after_request(response): return response -from app import views +from app.views import register_blueprints +from app.views import load_user +from app.views import load_user_from_request + +register_blueprints(app) + +from app.post_init import run_post_init + +try: + + run_post_init(development=app.config['DEVELOPMENT']) + +except Exception as e: + app.logger.exception('Post init failed. IRIS not started') + raise e + +lm.user_loader(load_user) +lm.request_loader(load_user_from_request) diff --git a/source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py b/source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py index 383537e4f..7d711ae9d 100644 --- a/source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py +++ b/source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. from app.alembic.alembic_utils import _table_has_column -from app.models import CompromiseStatus +from app.models.models import CompromiseStatus revision = '4ecdfcb34f7c' down_revision = 'a929ef458490' diff --git a/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py b/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py index 3c2a56b60..55ca24e10 100644 --- a/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py +++ b/source/app/alembic/versions/d5a720d1b99b_add_alerts_indexes.py @@ -5,10 +5,8 @@ Create Date: 2024-10-28 12:54:22.782313 """ -import sqlalchemy as sa from alembic import op -from sqlalchemy import and_, or_, text -from sqlalchemy.orm import Session +from sqlalchemy import text from app.alembic.alembic_utils import _has_table, index_exists diff --git a/source/app/blueprints/access_controls.py b/source/app/blueprints/access_controls.py index 3b76038f0..39de3e9ba 100644 --- a/source/app/blueprints/access_controls.py +++ b/source/app/blueprints/access_controls.py @@ -48,7 +48,7 @@ from app.iris_engine.access_control.utils import ac_fast_check_user_has_case_access from app.iris_engine.access_control.utils import ac_get_effective_permissions_of_user from app.iris_engine.utils.tracker import track_activity -from app.models import Cases +from app.models.cases import Cases from app.models.authorization import Permissions from app.models.authorization import CaseAccessLevel diff --git a/source/app/blueprints/pages/case/case_routes.py b/source/app/blueprints/pages/case/case_routes.py index 8b01f1c89..4c4b1f051 100644 --- a/source/app/blueprints/pages/case/case_routes.py +++ b/source/app/blueprints/pages/case/case_routes.py @@ -32,7 +32,7 @@ from app.forms import PipelinesCaseForm from app.iris_engine.access_control.utils import ac_get_all_access_level from app.iris_engine.module_handler.module_handler import list_available_pipelines -from app.models import CaseStatus +from app.models.models import CaseStatus from app.models.authorization import CaseAccessLevel from app.blueprints.access_controls import ac_case_requires diff --git a/source/app/blueprints/pages/case/case_timeline_routes.py b/source/app/blueprints/pages/case/case_timeline_routes.py index fac5680c9..4297acf54 100644 --- a/source/app/blueprints/pages/case/case_timeline_routes.py +++ b/source/app/blueprints/pages/case/case_timeline_routes.py @@ -38,7 +38,6 @@ from app.models.cases import CasesEvent from app.blueprints.access_controls import ac_case_requires from app.blueprints.responses import response_error -from app.schema.marshables import EventCategorySchema _EVENT_TAGS = ['Network', 'Server', 'ActiveDirectory', 'Computer', 'Malware', 'User Interaction'] diff --git a/source/app/blueprints/pages/dashboard/dashboard_routes.py b/source/app/blueprints/pages/dashboard/dashboard_routes.py index cdc289717..789c04206 100644 --- a/source/app/blueprints/pages/dashboard/dashboard_routes.py +++ b/source/app/blueprints/pages/dashboard/dashboard_routes.py @@ -19,24 +19,17 @@ from flask import Blueprint from flask import redirect from flask import render_template -from flask import session from flask import url_for from flask_login import current_user -from flask_login import logout_user from flask_wtf import FlaskForm from app import app -from app import db -from app import oidc_client from app.datamgmt.dashboard.dashboard_db import get_tasks_status from app.forms import CaseGlobalTaskForm from app.iris_engine.access_control.utils import ac_get_user_case_counts -from app.iris_engine.utils.tracker import track_activity from app.models.authorization import User from app.models.models import GlobalTasks -from app.blueprints.access_controls import ac_requires, is_authentication_oidc, not_authenticated_redirection_url - -from oic.oauth2.exception import GrantError +from app.blueprints.access_controls import ac_requires dashboard_blueprint = Blueprint( 'index', diff --git a/source/app/blueprints/pages/dim_tasks/dim_tasks.py b/source/app/blueprints/pages/dim_tasks/dim_tasks.py index f073f647d..423d89567 100644 --- a/source/app/blueprints/pages/dim_tasks/dim_tasks.py +++ b/source/app/blueprints/pages/dim_tasks/dim_tasks.py @@ -26,9 +26,10 @@ import app from app.models.authorization import CaseAccessLevel from app.models.authorization import Permissions -from app.blueprints.access_controls import ac_case_requires, ac_requires +from app.blueprints.access_controls import ac_case_requires +from app.blueprints.access_controls import ac_requires from app.blueprints.responses import response_error -from iris_interface.IrisInterfaceStatus import IIStatus +from app.business.dim_tasks import dim_tasks_get dim_tasks_blueprint = Blueprint( 'dim_tasks', @@ -54,49 +55,7 @@ def dim_index(caseid: int, url_redir): @ac_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access) def task_status(task_id, caseid, url_redir): if url_redir: - return response_error("Invalid request") + return response_error('Invalid request') - task = app.celery.AsyncResult(task_id) - - try: - tinfo = task.info - except AttributeError: - # Legacy task - task_info = { - 'Danger': 'This task was executed in a previous version of IRIS and the status cannot be read anymore.', - 'Note': 'All the data readable by the current IRIS version is displayed in the table.', - 'Additional information': 'The results of this tasks were stored in a pickled Class which does not exists ' - 'anymore in current IRIS version.' - } - return render_template("modal_task_info.html", data=task_info, task_id=task.id) - - task_info = { - 'Task ID': task_id, - 'Task finished on': task.date_done, - 'Task state': task.state.lower(), - 'Engine': task.name if task.name else "No engine. Unrecoverable shadow failure"} - - task_meta = task._get_task_meta() - - if task_meta.get('name') \ - and ('task_hook_wrapper' in task_meta.get('name') or 'pipeline_dispatcher' in task_meta.get('name')): - task_info['Module name'] = task_meta.get('kwargs').get('module_name') - task_info['Hook name'] = task_meta.get('kwargs').get('hook_name') - task_info['User'] = task_meta.get('kwargs').get('init_user') - task_info['Case ID'] = task_meta.get('kwargs').get('caseid') - - if isinstance(task.info, IIStatus): - success = task.info.is_success() - task_info['Logs'] = task.info.get_logs() - - else: - success = None - task_info['User'] = "Shadow Iris" - task_info['Logs'] = ['Task did not returned a valid IIStatus object'] - - if task_meta.get('traceback'): - task_info['Traceback'] = task.traceback - - task_info['Success'] = "Success" if success else "Failure" - - return render_template("modal_task_info.html", data=task_info, task_id=task.id) + task_info = dim_tasks_get(task_id) + return render_template('modal_task_info.html', data=task_info) diff --git a/source/app/blueprints/pages/dim_tasks/templates/modal_task_info.html b/source/app/blueprints/pages/dim_tasks/templates/modal_task_info.html index 6aec5cb24..d3fb9b686 100644 --- a/source/app/blueprints/pages/dim_tasks/templates/modal_task_info.html +++ b/source/app/blueprints/pages/dim_tasks/templates/modal_task_info.html @@ -8,7 +8,7 @@ {{ data[element] }} - {% endif %} + {% endif %} {% endif %} {% endfor %} diff --git a/source/app/blueprints/pages/login/login_routes.py b/source/app/blueprints/pages/login/login_routes.py index 58b472c50..8d76c21ef 100644 --- a/source/app/blueprints/pages/login/login_routes.py +++ b/source/app/blueprints/pages/login/login_routes.py @@ -188,7 +188,7 @@ def oidc_authorise(): if not user: log.warning(f"OIDC user {user_login} not found in database") if app.config.get("AUTHENTICATION_CREATE_USER_IF_NOT_EXIST") is False: - log.warning(f"Authentication is set to not create user if not exists") + log.warning('Authentication is set to not create user if not exists') track_activity( f"OIDC user {user_login} not found in database", ctx_less=True, diff --git a/source/app/blueprints/pages/manage/manage_case_templates_routes.py b/source/app/blueprints/pages/manage/manage_case_templates_routes.py index 94ef310e3..33f101fe5 100644 --- a/source/app/blueprints/pages/manage/manage_case_templates_routes.py +++ b/source/app/blueprints/pages/manage/manage_case_templates_routes.py @@ -22,7 +22,7 @@ from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id from app.forms import CaseTemplateForm, AddAssetForm -from app.models import CaseTemplate +from app.models.models import CaseTemplate from app.models.authorization import Permissions from app.blueprints.access_controls import ac_requires from app.blueprints.responses import response_error diff --git a/source/app/blueprints/pages/manage/manage_ioc_types_routes.py b/source/app/blueprints/pages/manage/manage_ioc_types_routes.py index 16594705e..479f908a4 100644 --- a/source/app/blueprints/pages/manage/manage_ioc_types_routes.py +++ b/source/app/blueprints/pages/manage/manage_ioc_types_routes.py @@ -22,7 +22,7 @@ from werkzeug.utils import redirect from app.forms import AddIocTypeForm -from app.models import IocType +from app.models.models import IocType from app.models.authorization import Permissions from app.blueprints.access_controls import ac_requires from app.blueprints.responses import response_error diff --git a/source/app/blueprints/rest/alerts_routes.py b/source/app/blueprints/rest/alerts_routes.py index 7c6376947..52e216825 100644 --- a/source/app/blueprints/rest/alerts_routes.py +++ b/source/app/blueprints/rest/alerts_routes.py @@ -31,8 +31,8 @@ from app.blueprints.rest.case_comments import case_comment_update from app.datamgmt.alerts.alerts_db import get_filtered_alerts from app.datamgmt.alerts.alerts_db import get_alert_by_id -from app.datamgmt.alerts.alerts_db import create_case_from_alert, \ - register_related_alerts, delete_related_alerts_cache +from app.datamgmt.alerts.alerts_db import create_case_from_alert +from app.datamgmt.alerts.alerts_db import delete_related_alerts_cache from app.datamgmt.alerts.alerts_db import merge_alert_in_case from app.datamgmt.alerts.alerts_db import unmerge_alert_from_case from app.datamgmt.alerts.alerts_db import cache_similar_alert @@ -50,7 +50,7 @@ from app.iris_engine.access_control.utils import ac_set_new_case_access from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models.alerts import AlertStatus, AlertSimilarity, Alert +from app.models.alerts import AlertStatus from app.models.authorization import Permissions from app.schema.marshables import AlertSchema from app.schema.marshables import CaseSchema @@ -399,7 +399,7 @@ def alerts_update_route(alert_id) -> Response: add_obj_history_entry(updated_alert, f"updated alert: {','.join(activity_data)}") else: track_activity(f"updated alert #{alert_id}", ctx_less=True) - add_obj_history_entry(updated_alert, f"updated alert") + add_obj_history_entry(updated_alert, 'updated alert') db.session.commit() diff --git a/source/app/blueprints/rest/case/case_assets_routes.py b/source/app/blueprints/rest/case/case_assets_routes.py index 393fbcc6c..913f680fb 100644 --- a/source/app/blueprints/rest/case/case_assets_routes.py +++ b/source/app/blueprints/rest/case/case_assets_routes.py @@ -51,7 +51,7 @@ from app.iris_engine.access_control.utils import ac_fast_check_current_user_has_case_access from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models import AnalysisStatus +from app.models.models import AnalysisStatus from app.models.authorization import CaseAccessLevel from app.schema.marshables import CaseAssetsSchema from app.schema.marshables import CommentSchema diff --git a/source/app/blueprints/rest/case/case_notes_routes.py b/source/app/blueprints/rest/case/case_notes_routes.py index 5cd6eaba8..38825affd 100644 --- a/source/app/blueprints/rest/case/case_notes_routes.py +++ b/source/app/blueprints/rest/case/case_notes_routes.py @@ -46,7 +46,7 @@ from app.datamgmt.states import get_notes_state from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models import Notes +from app.models.models import Notes from app.models.authorization import CaseAccessLevel from app.schema.marshables import CaseNoteDirectorySchema from app.schema.marshables import CaseNoteRevisionSchema diff --git a/source/app/blueprints/rest/case/case_routes.py b/source/app/blueprints/rest/case/case_routes.py index 95603ff16..b7767ca05 100644 --- a/source/app/blueprints/rest/case/case_routes.py +++ b/source/app/blueprints/rest/case/case_routes.py @@ -43,9 +43,9 @@ from app.iris_engine.access_control.utils import ac_fast_check_user_has_case_access from app.iris_engine.access_control.utils import ac_set_case_access_for_users from app.iris_engine.utils.tracker import track_activity -from app.models import CaseStatus -from app.models import ReviewStatusList -from app.models import UserActivity +from app.models.models import CaseStatus +from app.models.models import ReviewStatusList +from app.models.models import UserActivity from app.models.authorization import CaseAccessLevel from app.models.authorization import User from app.schema.marshables import TaskLogSchema diff --git a/source/app/blueprints/rest/case/case_timeline_routes.py b/source/app/blueprints/rest/case/case_timeline_routes.py index 25a405a90..dc69637ec 100644 --- a/source/app/blueprints/rest/case/case_timeline_routes.py +++ b/source/app/blueprints/rest/case/case_timeline_routes.py @@ -54,7 +54,7 @@ from app.iris_engine.utils.collab import collab_notify from app.iris_engine.utils.common import parse_bf_date_format from app.iris_engine.utils.tracker import track_activity -from app.models import CompromiseStatus +from app.models.models import CompromiseStatus from app.models.authorization import CaseAccessLevel from app.models.authorization import User from app.models.cases import CasesEvent diff --git a/source/app/blueprints/rest/dashboard_routes.py b/source/app/blueprints/rest/dashboard_routes.py index 7810b5a33..58f1d1b54 100644 --- a/source/app/blueprints/rest/dashboard_routes.py +++ b/source/app/blueprints/rest/dashboard_routes.py @@ -20,15 +20,17 @@ from datetime import datetime from datetime import timedelta -from flask import Blueprint, session, redirect +from flask import Blueprint from flask import request -from flask_login import current_user, logout_user +from flask_login import current_user -from app import db, app +from app import db +from app import app -from app import oidc_client from app.blueprints.rest.endpoints import endpoint_deprecated -from app.datamgmt.dashboard.dashboard_db import get_global_task, list_user_cases, list_user_reviews +from app.datamgmt.dashboard.dashboard_db import get_global_task +from app.datamgmt.dashboard.dashboard_db import list_user_cases +from app.datamgmt.dashboard.dashboard_db import list_user_reviews from app.datamgmt.dashboard.dashboard_db import get_tasks_status from app.datamgmt.dashboard.dashboard_db import list_global_tasks from app.datamgmt.dashboard.dashboard_db import list_user_tasks @@ -43,18 +45,14 @@ from app.schema.marshables import CaseTaskSchema from app.schema.marshables import CaseDetailsSchema from app.schema.marshables import GlobalTasksSchema -from app.blueprints.access_controls import ac_requires_case_identifier, is_authentication_oidc, \ - not_authenticated_redirection_url +from app.blueprints.access_controls import ac_requires_case_identifier from app.blueprints.access_controls import ac_api_requires from app.blueprints.responses import response_error from app.blueprints.responses import response_success -from oic.oauth2.exception import GrantError - log = app.logger -# CONTENT ------------------------------------------------ dashboard_rest_blueprint = Blueprint( 'dashboard_rest', __name__, diff --git a/source/app/blueprints/rest/dim_tasks_routes.py b/source/app/blueprints/rest/dim_tasks_routes.py index 599fc2040..882090220 100644 --- a/source/app/blueprints/rest/dim_tasks_routes.py +++ b/source/app/blueprints/rest/dim_tasks_routes.py @@ -23,24 +23,25 @@ from sqlalchemy import desc from app.iris_engine.module_handler.module_handler import call_modules_hook -from app.models import CeleryTaskMeta -from app.models import IrisHook -from app.models import IrisModule -from app.models import IrisModuleHook -from app.models import CaseAssets -from app.models import CaseReceivedFile -from app.models import CaseTasks -from app.models import Cases -from app.models import CasesEvent -from app.models import GlobalTasks -from app.models import Ioc -from app.models import Notes +from app.models.models import CeleryTaskMeta +from app.models.models import IrisHook +from app.models.models import IrisModule +from app.models.models import IrisModuleHook +from app.models.models import CaseAssets +from app.models.models import CaseReceivedFile +from app.models.models import CaseTasks +from app.models.cases import Cases +from app.models.cases import CasesEvent +from app.models.models import GlobalTasks +from app.models.models import Ioc +from app.models.models import Notes from app.models.alerts import Alert from app.models.authorization import CaseAccessLevel from app.blueprints.access_controls import ac_requires_case_identifier from app.blueprints.access_controls import ac_api_requires from app.blueprints.responses import response_error from app.blueprints.responses import response_success +from app.business.dim_tasks import dim_tasks_is_legacy from iris_interface.IrisInterfaceStatus import IIStatus dim_tasks_rest_blueprint = Blueprint('dim_tasks_rest', __name__) @@ -186,10 +187,7 @@ def list_dim_tasks(count): tkp = {'state': row.status, 'case': "Unknown", 'module': row.name, 'task_id': row.task_id, 'date_done': row.date_done, 'user': "Unknown"} - try: - _ = row.result - except AttributeError: - # Legacy task + if dim_tasks_is_legacy(row): data.append(tkp) continue diff --git a/source/app/blueprints/rest/manage/manage_case_templates_routes.py b/source/app/blueprints/rest/manage/manage_case_templates_routes.py index aab98e66d..0f4fa5835 100644 --- a/source/app/blueprints/rest/manage/manage_case_templates_routes.py +++ b/source/app/blueprints/rest/manage/manage_case_templates_routes.py @@ -27,7 +27,7 @@ from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id from app.datamgmt.manage.manage_case_templates_db import validate_case_template from app.datamgmt.manage.manage_case_templates_db import delete_case_template_by_id -from app.models import CaseTemplate +from app.models.models import CaseTemplate from app.models.authorization import Permissions from app.iris_engine.utils.tracker import track_activity from app.schema.marshables import CaseTemplateSchema diff --git a/source/app/blueprints/rest/manage/manage_ioc_types_routes.py b/source/app/blueprints/rest/manage/manage_ioc_types_routes.py index 6f5a07d9c..bbe9a62d3 100644 --- a/source/app/blueprints/rest/manage/manage_ioc_types_routes.py +++ b/source/app/blueprints/rest/manage/manage_ioc_types_routes.py @@ -24,8 +24,8 @@ from app.datamgmt.case.case_iocs_db import get_ioc_types_list from app.datamgmt.manage.manage_case_objs import search_ioc_type_by_name from app.iris_engine.utils.tracker import track_activity -from app.models import Ioc -from app.models import IocType +from app.models.models import Ioc +from app.models.models import IocType from app.models.authorization import Permissions from app.schema.marshables import IocTypeSchema from app.blueprints.access_controls import ac_api_requires diff --git a/source/app/blueprints/rest/manage/manage_tlps_routes.py b/source/app/blueprints/rest/manage/manage_tlps_routes.py index a72ec43f2..28203b28d 100644 --- a/source/app/blueprints/rest/manage/manage_tlps_routes.py +++ b/source/app/blueprints/rest/manage/manage_tlps_routes.py @@ -18,7 +18,7 @@ from flask import Blueprint -from app.models import Tlp +from app.models.models import Tlp from app.blueprints.access_controls import ac_api_requires from app.blueprints.responses import response_error from app.blueprints.responses import response_success diff --git a/source/app/blueprints/rest/reports_route.py b/source/app/blueprints/rest/reports_route.py index 94d7b87ce..a8052b317 100644 --- a/source/app/blueprints/rest/reports_route.py +++ b/source/app/blueprints/rest/reports_route.py @@ -29,7 +29,7 @@ from app.iris_engine.reporter.reporter import IrisMakeMdReport from app.iris_engine.utils.tracker import track_activity -from app.models import CaseTemplateReport +from app.models.models import CaseTemplateReport from app.models.authorization import CaseAccessLevel from app.util import FileRemover diff --git a/source/app/blueprints/rest/search_routes.py b/source/app/blueprints/rest/search_routes.py index 7da5e4410..395af5fe2 100644 --- a/source/app/blueprints/rest/search_routes.py +++ b/source/app/blueprints/rest/search_routes.py @@ -21,7 +21,7 @@ from sqlalchemy import and_ from app.iris_engine.utils.tracker import track_activity -from app.models import Comments +from app.models.models import Comments from app.models.authorization import Permissions from app.models.cases import Cases from app.models.models import Client diff --git a/source/app/blueprints/rest/v2/auth/__init__.py b/source/app/blueprints/rest/v2/auth/__init__.py index 907b27a42..e17f8d3b9 100644 --- a/source/app/blueprints/rest/v2/auth/__init__.py +++ b/source/app/blueprints/rest/v2/auth/__init__.py @@ -16,23 +16,26 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from flask import Blueprint, session -from flask import redirect, url_for +from flask import Blueprint +from flask import session +from flask import redirect +from flask import url_for from flask import request -from flask_login import current_user, logout_user +from flask_login import current_user +from flask_login import logout_user from oic.oauth2.exception import GrantError from app import app from app import db from app import oidc_client from app.blueprints.access_controls import is_authentication_ldap -from app.blueprints.access_controls import is_authentication_oidc, \ - not_authenticated_redirection_url +from app.blueprints.access_controls import is_authentication_oidc +from app.blueprints.access_controls import not_authenticated_redirection_url from app.blueprints.rest.endpoints import response_api_error from app.blueprints.rest.endpoints import response_api_success -from app.business.auth import validate_ldap_login, validate_local_login +from app.business.auth import validate_ldap_login +from app.business.auth import validate_local_login from app.iris_engine.utils.tracker import track_activity -from app.models.authorization import User from app.schema.marshables import UserSchema diff --git a/source/app/business/assets.py b/source/app/business/assets.py index 84e800f4d..b2da01ff4 100644 --- a/source/app/business/assets.py +++ b/source/app/business/assets.py @@ -24,8 +24,11 @@ from app.datamgmt.case.case_db import get_case_client_id from app.datamgmt.manage.manage_users_db import get_user_cases_fast from app.datamgmt.states import get_assets_state -from app.models import CaseAssets -from app.datamgmt.case.case_assets_db import get_asset, get_assets, get_assets_ioc_links, get_similar_assets +from app.datamgmt.case.case_assets_db import get_assets +from app.datamgmt.case.case_assets_db import get_assets_ioc_links +from app.datamgmt.case.case_assets_db import get_similar_assets +from app.models.models import CaseAssets +from app.datamgmt.case.case_assets_db import get_asset from app.datamgmt.case.case_assets_db import case_assets_db_exists from app.datamgmt.case.case_assets_db import create_asset from app.datamgmt.case.case_assets_db import set_ioc_links diff --git a/source/app/business/auth.py b/source/app/business/auth.py index 0f4ec12a4..2472d2258 100644 --- a/source/app/business/auth.py +++ b/source/app/business/auth.py @@ -1,3 +1,21 @@ +# IRIS Source Code +# Copyright (C) 2024 - DFIR-IRIS +# contact@dfir-iris.org +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + from urllib.parse import urlsplit from flask import session, redirect, url_for, request @@ -9,7 +27,7 @@ from app.iris_engine.access_control.ldap_handler import ldap_authenticate from app.iris_engine.access_control.utils import ac_get_effective_permissions_of_user from app.iris_engine.utils.tracker import track_activity -from app.models import Cases +from app.models.cases import Cases from app.schema.marshables import UserSchema log = app.logger diff --git a/source/app/business/cases.py b/source/app/business/cases.py index d9b115596..799af0763 100644 --- a/source/app/business/cases.py +++ b/source/app/business/cases.py @@ -30,7 +30,7 @@ from app.util import add_obj_history_entry from app.schema.marshables import CaseSchema -from app.models import ReviewStatusList +from app.models.models import ReviewStatusList from app.business.errors import BusinessProcessingError from app.business.iocs import iocs_exports_to_json diff --git a/source/app/business/dim_tasks.py b/source/app/business/dim_tasks.py new file mode 100644 index 000000000..4ca2b8645 --- /dev/null +++ b/source/app/business/dim_tasks.py @@ -0,0 +1,82 @@ +# IRIS Source Code +# Copyright (C) 2024 - DFIR-IRIS +# contact@dfir-iris.org +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +from app import celery +from iris_interface.IrisInterfaceStatus import IIStatus + + +def _get_engine_name(task): + if not task.name: + return 'No engine. Unrecoverable shadow failure' + return task.name + +def _get_success(task_result: IIStatus): + if task_result.is_success(): + return 'Success' + else: + return 'Failure' + +def dim_tasks_is_legacy(task): + try: + _ = task.date_done + return False + except AttributeError: + return True + +def dim_tasks_get(task_identifier): + task = celery.AsyncResult(task_identifier) + if dim_tasks_is_legacy(task): + return { + 'Danger': 'This task was executed in a previous version of IRIS and the status cannot be read anymore.', + 'Note': 'All the data readable by the current IRIS version is displayed in the table.', + 'Additional information': 'The results of this tasks were stored in a pickled Class which does not exists ' + 'anymore in current IRIS version.' + } + + engine_name = _get_engine_name(task) + user = None + module_name = None + hook_name = None + case_identifier = None + if task.name and ('task_hook_wrapper' in task.name or 'pipeline_dispatcher' in task.name): + module_name = task.kwargs.get('module_name') + hook_name = task.kwargs.get('hook_name') + user = task.kwargs.get('init_user') + case_identifier = task.kwargs.get('caseid') + + if isinstance(task.info, IIStatus): + success = _get_success(task.info) + logs = task.info.get_logs() + else: + success = 'Failure' + user = 'Shadow Iris' + logs = ['Task did not returned a valid IIStatus object'] + + return { + 'Task ID': task_identifier, + 'Task finished on': task.date_done, + 'Task state': task.state.lower(), + 'Engine': engine_name, + 'Module name': module_name, + 'Hook name': hook_name, + 'Case ID': case_identifier, + 'Success': success, + 'User': user, + 'Logs': logs, + 'Traceback': task.traceback + } diff --git a/source/app/business/iocs.py b/source/app/business/iocs.py index 3374695da..8473501c5 100644 --- a/source/app/business/iocs.py +++ b/source/app/business/iocs.py @@ -20,7 +20,7 @@ from marshmallow.exceptions import ValidationError from app import db -from app.models import Ioc +from app.models.models import Ioc from app.datamgmt.case.case_iocs_db import add_ioc from app.datamgmt.case.case_iocs_db import case_iocs_db_exists from app.datamgmt.case.case_iocs_db import check_ioc_type_id diff --git a/source/app/business/notes.py b/source/app/business/notes.py index 96117d433..c3647f5e6 100644 --- a/source/app/business/notes.py +++ b/source/app/business/notes.py @@ -25,7 +25,7 @@ from app.datamgmt.case.case_notes_db import get_note from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models import NoteRevisions +from app.models.models import NoteRevisions from app.models.authorization import User from app.schema.marshables import CaseNoteSchema from app.util import add_obj_history_entry diff --git a/source/app/business/tasks.py b/source/app/business/tasks.py index cd6230444..9a026d537 100644 --- a/source/app/business/tasks.py +++ b/source/app/business/tasks.py @@ -29,7 +29,7 @@ from app.datamgmt.states import update_tasks_state from app.iris_engine.module_handler.module_handler import call_modules_hook from app.iris_engine.utils.tracker import track_activity -from app.models import CaseTasks +from app.models.models import CaseTasks from app.schema.marshables import CaseTaskSchema from app.business.errors import BusinessProcessingError from app.business.errors import ObjectNotFoundError diff --git a/source/app/datamgmt/activities/activities_db.py b/source/app/datamgmt/activities/activities_db.py index c1fb0c9b5..f27134a00 100644 --- a/source/app/datamgmt/activities/activities_db.py +++ b/source/app/datamgmt/activities/activities_db.py @@ -19,7 +19,7 @@ from sqlalchemy import and_ from sqlalchemy import desc -from app.models import Cases +from app.models.cases import Cases from app.models.authorization import User from app.models.models import UserActivity diff --git a/source/app/datamgmt/alerts/alerts_db.py b/source/app/datamgmt/alerts/alerts_db.py index 84b379476..015daae27 100644 --- a/source/app/datamgmt/alerts/alerts_db.py +++ b/source/app/datamgmt/alerts/alerts_db.py @@ -20,11 +20,9 @@ import json from datetime import datetime, timedelta from flask_login import current_user -from functools import reduce from sqlalchemy import desc, asc, func, tuple_, or_, not_, and_ from sqlalchemy.orm import aliased, make_transient, selectinload -from typing import List, Tuple, Dict - +from typing import List, Tuple import app from app import db from app.datamgmt.case.case_assets_db import create_asset, set_ioc_links, get_unspecified_analysis_status_id @@ -32,17 +30,33 @@ from app.datamgmt.case.case_iocs_db import add_ioc from app.datamgmt.manage.manage_access_control_db import get_user_clients_id from app.datamgmt.manage.manage_case_state_db import get_case_state_by_name -from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id, \ - case_template_post_modifier +from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id +from app.datamgmt.manage.manage_case_templates_db import case_template_post_modifier from app.datamgmt.states import update_timeline_state + from app.iris_engine.access_control.utils import ac_current_user_has_permission +from app.models.cases import Cases +from app.models.models import Client +from app.models.models import EventCategory +from app.models.models import Tags +from app.models.models import AssetsType +from app.models.models import Comments +from app.models.models import CaseAssets +from app.models.models import alert_assets_association +from app.models.models import alert_iocs_association +from app.models.models import Ioc +from app.models.alerts import Alert +from app.models.alerts import AlertStatus +from app.models.alerts import AlertCaseAssociation +from app.models.alerts import SimilarAlertsCache +from app.models.alerts import AlertResolutionStatus +from app.models.authorization import Permissions from app.iris_engine.utils.common import parse_bf_date_format -from app.models import Cases, EventCategory, Tags, AssetsType, Comments, CaseAssets, alert_assets_association, \ - alert_iocs_association, Ioc, Client -from app.models.alerts import Alert, AlertStatus, AlertCaseAssociation, SimilarAlertsCache, AlertResolutionStatus, \ - AlertSimilarity, Severity -from app.models.authorization import Permissions, User -from app.schema.marshables import EventSchema, AlertSchema +from app.models.alerts import AlertSimilarity +from app.models.alerts import Severity +from app.models.authorization import User +from app.schema.marshables import EventSchema +from app.schema.marshables import AlertSchema from app.util import add_obj_history_entry diff --git a/source/app/datamgmt/case/case_assets_db.py b/source/app/datamgmt/case/case_assets_db.py index e84460ecd..3d668e551 100644 --- a/source/app/datamgmt/case/case_assets_db.py +++ b/source/app/datamgmt/case/case_assets_db.py @@ -24,17 +24,18 @@ from app import db, app from app.datamgmt.states import update_assets_state -from app.models import AnalysisStatus, CaseStatus -from app.models import AssetComments -from app.models import AssetsType -from app.models import CaseAssets -from app.models import CaseEventsAssets -from app.models import Cases -from app.models import Comments -from app.models import CompromiseStatus -from app.models import Ioc -from app.models import IocAssetLink -from app.models import IocType +from app.models.models import AnalysisStatus +from app.models.models import CaseStatus +from app.models.models import AssetComments +from app.models.models import AssetsType +from app.models.models import CaseAssets +from app.models.models import CaseEventsAssets +from app.models.cases import Cases +from app.models.models import Comments +from app.models.models import CompromiseStatus +from app.models.models import Ioc +from app.models.models import IocAssetLink +from app.models.models import IocType from app.models.authorization import User diff --git a/source/app/datamgmt/case/case_comments.py b/source/app/datamgmt/case/case_comments.py index e34410619..890f2ba8c 100644 --- a/source/app/datamgmt/case/case_comments.py +++ b/source/app/datamgmt/case/case_comments.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from app.models import Comments +from app.models.models import Comments def get_case_comment(comment_id, caseid): diff --git a/source/app/datamgmt/case/case_events_db.py b/source/app/datamgmt/case/case_events_db.py index 6be300d90..c124a907a 100644 --- a/source/app/datamgmt/case/case_events_db.py +++ b/source/app/datamgmt/case/case_events_db.py @@ -20,18 +20,18 @@ from app import db from app.datamgmt.states import update_timeline_state -from app.models import AssetsType -from app.models import CaseAssets -from app.models import CaseEventCategory -from app.models import CaseEventsAssets -from app.models import CaseEventsIoc -from app.models import CasesEvent -from app.models import Comments -from app.models import EventCategory -from app.models import EventComments -from app.models import Ioc -from app.models import IocAssetLink -from app.models import IocType +from app.models.models import AssetsType +from app.models.models import CaseAssets +from app.models.models import CaseEventCategory +from app.models.models import CaseEventsAssets +from app.models.models import CaseEventsIoc +from app.models.cases import CasesEvent +from app.models.models import Comments +from app.models.models import EventCategory +from app.models.models import EventComments +from app.models.models import Ioc +from app.models.models import IocAssetLink +from app.models.models import IocType from app.models.authorization import User diff --git a/source/app/datamgmt/case/case_iocs_db.py b/source/app/datamgmt/case/case_iocs_db.py index 47dc5819f..355525a9a 100644 --- a/source/app/datamgmt/case/case_iocs_db.py +++ b/source/app/datamgmt/case/case_iocs_db.py @@ -25,13 +25,13 @@ from app import app from app.datamgmt.states import update_ioc_state from app.iris_engine.access_control.utils import ac_get_fast_user_cases_access -from app.models import Cases -from app.models import Client -from app.models import Comments -from app.models import Ioc -from app.models import IocComments -from app.models import IocType -from app.models import Tlp +from app.models.cases import Cases +from app.models.models import Client +from app.models.models import Comments +from app.models.models import Ioc +from app.models.models import IocComments +from app.models.models import IocType +from app.models.models import Tlp from app.models.authorization import User from app.models.authorization import UserCaseEffectiveAccess from app.models.authorization import CaseAccessLevel diff --git a/source/app/datamgmt/case/case_notes_db.py b/source/app/datamgmt/case/case_notes_db.py index 5871c1e25..9a5ad5140 100644 --- a/source/app/datamgmt/case/case_notes_db.py +++ b/source/app/datamgmt/case/case_notes_db.py @@ -21,11 +21,13 @@ from app import db from app.datamgmt.manage.manage_attribute_db import get_default_custom_attributes from app.datamgmt.states import update_notes_state -from app.models import Comments, NoteDirectory, NoteRevisions -from app.models import Notes -from app.models import NotesComments -from app.models import NotesGroup -from app.models import NotesGroupLink +from app.models.models import Comments +from app.models.models import NoteDirectory +from app.models.models import NoteRevisions +from app.models.models import Notes +from app.models.models import NotesComments +from app.models.models import NotesGroup +from app.models.models import NotesGroupLink from app.models.authorization import User diff --git a/source/app/datamgmt/case/case_rfiles_db.py b/source/app/datamgmt/case/case_rfiles_db.py index 3425972a6..950cf3704 100644 --- a/source/app/datamgmt/case/case_rfiles_db.py +++ b/source/app/datamgmt/case/case_rfiles_db.py @@ -24,9 +24,9 @@ from app import db from app.datamgmt.manage.manage_attribute_db import get_default_custom_attributes from app.datamgmt.states import update_evidences_state -from app.models import CaseReceivedFile -from app.models import Comments -from app.models import EvidencesComments +from app.models.models import CaseReceivedFile +from app.models.models import Comments +from app.models.models import EvidencesComments from app.models.authorization import User diff --git a/source/app/datamgmt/case/case_tasks_db.py b/source/app/datamgmt/case/case_tasks_db.py index fa0cf38fc..9f78c9278 100644 --- a/source/app/datamgmt/case/case_tasks_db.py +++ b/source/app/datamgmt/case/case_tasks_db.py @@ -18,18 +18,19 @@ from datetime import datetime from flask_login import current_user -from sqlalchemy import desc, and_ +from sqlalchemy import desc +from sqlalchemy import and_ from app import db from app.datamgmt.manage.manage_attribute_db import get_default_custom_attributes from app.datamgmt.manage.manage_users_db import get_users_list_restricted_from_case from app.datamgmt.states import update_tasks_state -from app.models import CaseTasks -from app.models import TaskAssignee -from app.models import Cases -from app.models import Comments -from app.models import TaskComments -from app.models import TaskStatus +from app.models.models import CaseTasks +from app.models.models import TaskAssignee +from app.models.cases import Cases +from app.models.models import Comments +from app.models.models import TaskComments +from app.models.models import TaskStatus from app.models.authorization import User diff --git a/source/app/datamgmt/client/client_db.py b/source/app/datamgmt/client/client_db.py index 7679c5e74..b81bae752 100644 --- a/source/app/datamgmt/client/client_db.py +++ b/source/app/datamgmt/client/client_db.py @@ -16,16 +16,18 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import marshmallow -from sqlalchemy import func, and_ +from sqlalchemy import func +from sqlalchemy import and_ from typing import List from app import db from app.datamgmt.exceptions.ElementExceptions import ElementInUseException from app.datamgmt.exceptions.ElementExceptions import ElementNotFoundException -from app.models import Cases -from app.models import Client -from app.models import Contact -from app.models.authorization import User, UserClient +from app.models.cases import Cases +from app.models.models import Client +from app.models.models import Contact +from app.models.authorization import User +from app.models.authorization import UserClient from app.schema.marshables import ContactSchema from app.schema.marshables import CustomerSchema diff --git a/source/app/datamgmt/context/context_db.py b/source/app/datamgmt/context/context_db.py index 52f262b15..27afcd233 100644 --- a/source/app/datamgmt/context/context_db.py +++ b/source/app/datamgmt/context/context_db.py @@ -15,11 +15,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from sqlalchemy import and_, case, or_, asc +from sqlalchemy import and_ +from sqlalchemy import case +from sqlalchemy import or_ +from sqlalchemy import asc from sqlalchemy import desc -from app.models import Cases -from app.models import Client +from app.models.cases import Cases +from app.models.models import Client from app.models.authorization import CaseAccessLevel from app.models.authorization import UserCaseEffectiveAccess from app.datamgmt.authorization import has_deny_all_access_level diff --git a/source/app/datamgmt/dashboard/dashboard_db.py b/source/app/datamgmt/dashboard/dashboard_db.py index dfd458a13..6cff16fd4 100644 --- a/source/app/datamgmt/dashboard/dashboard_db.py +++ b/source/app/datamgmt/dashboard/dashboard_db.py @@ -20,10 +20,12 @@ from sqlalchemy import desc from app import db -from app.models import CaseTasks, TaskAssignee, ReviewStatus -from app.models import Cases -from app.models import GlobalTasks -from app.models import TaskStatus +from app.models.models import CaseTasks +from app.models.models import TaskAssignee +from app.models.models import ReviewStatus +from app.models.cases import Cases +from app.models.models import GlobalTasks +from app.models.models import TaskStatus from app.models.authorization import User diff --git a/source/app/datamgmt/datastore/datastore_db.py b/source/app/datamgmt/datastore/datastore_db.py index 5135e06fa..fa6b85f19 100644 --- a/source/app/datamgmt/datastore/datastore_db.py +++ b/source/app/datamgmt/datastore/datastore_db.py @@ -27,12 +27,12 @@ from app import app from app import db -from app.models import CaseReceivedFile -from app.models import DataStoreFile -from app.models import DataStorePath -from app.models import Ioc -from app.models import IocType -from app.models import Tlp +from app.models.models import CaseReceivedFile +from app.models.models import DataStoreFile +from app.models.models import DataStorePath +from app.models.models import Ioc +from app.models.models import IocType +from app.models.models import Tlp def datastore_get_root(cid): diff --git a/source/app/datamgmt/filters/filters_db.py b/source/app/datamgmt/filters/filters_db.py index 759e1d79c..56f828d97 100644 --- a/source/app/datamgmt/filters/filters_db.py +++ b/source/app/datamgmt/filters/filters_db.py @@ -1,7 +1,7 @@ from flask_login import current_user from sqlalchemy import and_ -from app.models import SavedFilter +from app.models.models import SavedFilter def get_filter_by_id(filter_id): diff --git a/source/app/datamgmt/iris_engine/evidence_storage.py b/source/app/datamgmt/iris_engine/evidence_storage.py index a65b28a27..21ce14d14 100644 --- a/source/app/datamgmt/iris_engine/evidence_storage.py +++ b/source/app/datamgmt/iris_engine/evidence_storage.py @@ -17,7 +17,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from app.datamgmt.case.case_rfiles_db import add_rfile -from app.models import CaseReceivedFile +from app.models.models import CaseReceivedFile class EvidenceStorage(object): diff --git a/source/app/datamgmt/iris_engine/modules_db.py b/source/app/datamgmt/iris_engine/modules_db.py index 503f5980a..1abde96ae 100644 --- a/source/app/datamgmt/iris_engine/modules_db.py +++ b/source/app/datamgmt/iris_engine/modules_db.py @@ -20,9 +20,9 @@ from flask_login import current_user from app import db, app -from app.models import IrisHook -from app.models import IrisModule -from app.models import IrisModuleHook +from app.models.models import IrisHook +from app.models.models import IrisModule +from app.models.models import IrisModuleHook from app.models.authorization import User log = app.logger diff --git a/source/app/datamgmt/manage/manage_access_control_db.py b/source/app/datamgmt/manage/manage_access_control_db.py index c5b7d9132..a97db33a8 100644 --- a/source/app/datamgmt/manage/manage_access_control_db.py +++ b/source/app/datamgmt/manage/manage_access_control_db.py @@ -16,7 +16,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from app import ac_current_user_has_permission -from app.models import Cases +from app.models.cases import Cases from app.models.authorization import Group from app.models.authorization import UserClient from app.models.authorization import Permissions diff --git a/source/app/datamgmt/manage/manage_assets_db.py b/source/app/datamgmt/manage/manage_assets_db.py index c9e8b1a82..1daad63f1 100644 --- a/source/app/datamgmt/manage/manage_assets_db.py +++ b/source/app/datamgmt/manage/manage_assets_db.py @@ -1,10 +1,32 @@ +# IRIS Source Code +# Copyright (C) 2024 - DFIR-IRIS +# contact@dfir-iris.org +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + from flask_login import current_user -from sqlalchemy import and_, desc, asc +from sqlalchemy import and_ +from sqlalchemy import desc +from sqlalchemy import asc from functools import reduce import app from app.datamgmt.manage.manage_cases_db import user_list_cases_view -from app.models import CaseAssets, Client, Cases +from app.models.models import CaseAssets +from app.models.models import Client +from app.models.cases import Cases def get_filtered_assets(case_id=None, diff --git a/source/app/datamgmt/manage/manage_attribute_db.py b/source/app/datamgmt/manage/manage_attribute_db.py index 2237f6c1e..86924f10f 100644 --- a/source/app/datamgmt/manage/manage_attribute_db.py +++ b/source/app/datamgmt/manage/manage_attribute_db.py @@ -19,16 +19,17 @@ import logging as logger from sqlalchemy.orm.attributes import flag_modified -from app import db, app -from app.models import CaseAssets -from app.models import CaseReceivedFile -from app.models import CaseTasks -from app.models import Cases -from app.models import CasesEvent -from app.models import Client -from app.models import CustomAttribute -from app.models import Ioc -from app.models import Notes +from app import db +from app import app +from app.models.models import CaseAssets +from app.models.models import CaseReceivedFile +from app.models.models import CaseTasks +from app.models.cases import Cases +from app.models.cases import CasesEvent +from app.models.models import Client +from app.models.models import CustomAttribute +from app.models.models import Ioc +from app.models.models import Notes log = logger.getLogger(__name__) diff --git a/source/app/datamgmt/manage/manage_case_classifications_db.py b/source/app/datamgmt/manage/manage_case_classifications_db.py index 5f0adcece..09eb1fca8 100644 --- a/source/app/datamgmt/manage/manage_case_classifications_db.py +++ b/source/app/datamgmt/manage/manage_case_classifications_db.py @@ -17,7 +17,7 @@ from sqlalchemy import func from typing import List -from app.models import CaseClassification +from app.models.models import CaseClassification def get_case_classifications_list() -> List[dict]: diff --git a/source/app/datamgmt/manage/manage_case_objs.py b/source/app/datamgmt/manage/manage_case_objs.py index 014d77658..8ed653fcc 100644 --- a/source/app/datamgmt/manage/manage_case_objs.py +++ b/source/app/datamgmt/manage/manage_case_objs.py @@ -17,7 +17,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from sqlalchemy import func -from app.models import AnalysisStatus, IocType, AssetsType, EventCategory +from app.models.models import AnalysisStatus +from app.models.models import IocType +from app.models.models import AssetsType +from app.models.models import EventCategory def search_analysis_status_by_name(name: str, exact_match: bool = False) -> AnalysisStatus: diff --git a/source/app/datamgmt/manage/manage_case_templates_db.py b/source/app/datamgmt/manage/manage_case_templates_db.py index e98864d06..99691dde7 100644 --- a/source/app/datamgmt/manage/manage_case_templates_db.py +++ b/source/app/datamgmt/manage/manage_case_templates_db.py @@ -14,17 +14,26 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + import marshmallow from datetime import datetime -from typing import List, Optional, Union +from typing import List +from typing import Optional +from typing import Union from app import db from app.datamgmt.case.case_tasks_db import add_task from app.datamgmt.manage.manage_case_classifications_db import get_case_classification_by_name from app.iris_engine.module_handler.module_handler import call_modules_hook -from app.models import CaseTemplate, Cases, Tags, NoteDirectory +from app.models.models import CaseTemplate +from app.models.cases import Cases +from app.models.models import Tags +from app.models.models import NoteDirectory from app.models.authorization import User -from app.schema.marshables import CaseSchema, CaseTaskSchema, CaseNoteDirectorySchema, CaseNoteSchema +from app.schema.marshables import CaseSchema +from app.schema.marshables import CaseTaskSchema +from app.schema.marshables import CaseNoteDirectorySchema +from app.schema.marshables import CaseNoteSchema def get_case_templates_list() -> List[dict]: diff --git a/source/app/datamgmt/manage/manage_cases_db.py b/source/app/datamgmt/manage/manage_cases_db.py index 491ac7a17..6b80fec52 100644 --- a/source/app/datamgmt/manage/manage_cases_db.py +++ b/source/app/datamgmt/manage/manage_cases_db.py @@ -33,28 +33,29 @@ from app.datamgmt.manage.manage_case_state_db import get_case_state_by_name from app.datamgmt.authorization import has_deny_all_access_level from app.datamgmt.states import delete_case_states -from app.models import CaseAssets, NoteRevisions -from app.models import CaseClassification -from app.models import alert_assets_association -from app.models import CaseStatus -from app.models import TaskAssignee -from app.models import NoteDirectory -from app.models import Tags -from app.models import CaseEventCategory -from app.models import CaseEventsAssets -from app.models import CaseEventsIoc -from app.models import CaseReceivedFile -from app.models import CaseTasks -from app.models import Cases -from app.models import CasesEvent -from app.models import Client -from app.models import DataStoreFile -from app.models import DataStorePath -from app.models import IocAssetLink -from app.models import Notes -from app.models import NotesGroup -from app.models import NotesGroupLink -from app.models import UserActivity +from app.models.models import CaseAssets +from app.models.models import NoteRevisions +from app.models.models import CaseClassification +from app.models.models import alert_assets_association +from app.models.models import CaseStatus +from app.models.models import TaskAssignee +from app.models.models import NoteDirectory +from app.models.models import Tags +from app.models.models import CaseEventCategory +from app.models.models import CaseEventsAssets +from app.models.models import CaseEventsIoc +from app.models.models import CaseReceivedFile +from app.models.models import CaseTasks +from app.models.cases import Cases +from app.models.cases import CasesEvent +from app.models.models import Client +from app.models.models import DataStoreFile +from app.models.models import DataStorePath +from app.models.models import IocAssetLink +from app.models.models import Notes +from app.models.models import NotesGroup +from app.models.models import NotesGroupLink +from app.models.models import UserActivity from app.models.alerts import AlertCaseAssociation from app.models.authorization import CaseAccessLevel from app.models.authorization import GroupCaseAccess diff --git a/source/app/datamgmt/manage/manage_evidence_types_db.py b/source/app/datamgmt/manage/manage_evidence_types_db.py index 2184d9f1f..33c2b11a1 100644 --- a/source/app/datamgmt/manage/manage_evidence_types_db.py +++ b/source/app/datamgmt/manage/manage_evidence_types_db.py @@ -17,7 +17,8 @@ from sqlalchemy import func from typing import List -from app.models import EvidenceTypes, CaseReceivedFile +from app.models.models import EvidenceTypes +from app.models.models import CaseReceivedFile def get_evidence_types_list() -> List[dict]: diff --git a/source/app/datamgmt/manage/manage_groups_db.py b/source/app/datamgmt/manage/manage_groups_db.py index 3a197035b..3d3eedb21 100644 --- a/source/app/datamgmt/manage/manage_groups_db.py +++ b/source/app/datamgmt/manage/manage_groups_db.py @@ -24,7 +24,7 @@ from app.iris_engine.access_control.utils import ac_access_level_to_list from app.iris_engine.access_control.utils import ac_auto_update_user_effective_access from app.iris_engine.access_control.utils import ac_permission_to_list -from app.models import Cases +from app.models.cases import Cases from app.models.authorization import Group from app.models.authorization import GroupCaseAccess from app.models.authorization import User diff --git a/source/app/datamgmt/manage/manage_srv_settings_db.py b/source/app/datamgmt/manage/manage_srv_settings_db.py index 9dcd8dee2..a7e77b282 100644 --- a/source/app/datamgmt/manage/manage_srv_settings_db.py +++ b/source/app/datamgmt/manage/manage_srv_settings_db.py @@ -1,7 +1,7 @@ from sqlalchemy import text from app import db -from app.models import ServerSettings +from app.models.models import ServerSettings from app.schema.marshables import ServerSettingsSchema diff --git a/source/app/datamgmt/manage/manage_tags_db.py b/source/app/datamgmt/manage/manage_tags_db.py index 170bb4271..c472be086 100644 --- a/source/app/datamgmt/manage/manage_tags_db.py +++ b/source/app/datamgmt/manage/manage_tags_db.py @@ -1,9 +1,11 @@ from functools import reduce -from sqlalchemy import and_, desc, asc +from sqlalchemy import and_ +from sqlalchemy import desc +from sqlalchemy import asc import app -from app.models import Tags +from app.models.models import Tags def get_filtered_tags(tag_title=None, diff --git a/source/app/datamgmt/manage/manage_users_db.py b/source/app/datamgmt/manage/manage_users_db.py index 7628f4c7c..03f537c0c 100644 --- a/source/app/datamgmt/manage/manage_users_db.py +++ b/source/app/datamgmt/manage/manage_users_db.py @@ -31,7 +31,9 @@ from app.iris_engine.access_control.utils import ac_get_detailed_effective_permissions_from_groups from app.iris_engine.access_control.utils import ac_remove_case_access_from_user from app.iris_engine.access_control.utils import ac_set_case_access_for_user -from app.models import Cases, Client, UserActivity +from app.models.models import UserActivity +from app.models.cases import Cases +from app.models.models import Client from app.models.authorization import CaseAccessLevel, UserClient from app.models.authorization import Group from app.models.authorization import Organisation diff --git a/source/app/datamgmt/overview/overview_db.py b/source/app/datamgmt/overview/overview_db.py index 3ecff5148..62a097b08 100644 --- a/source/app/datamgmt/overview/overview_db.py +++ b/source/app/datamgmt/overview/overview_db.py @@ -20,7 +20,7 @@ from app.datamgmt.case.case_tasks_db import get_tasks_cases_mapping from app.datamgmt.manage.manage_cases_db import user_list_cases_view -from app.models import Cases +from app.models.cases import Cases from app.schema.marshables import CaseDetailsSchema diff --git a/source/app/datamgmt/reporter/report_db.py b/source/app/datamgmt/reporter/report_db.py index 13a269c36..8272724c0 100644 --- a/source/app/datamgmt/reporter/report_db.py +++ b/source/app/datamgmt/reporter/report_db.py @@ -22,26 +22,26 @@ from app.datamgmt.case.case_notes_db import get_notes_from_group from app.datamgmt.case.case_notes_db import get_case_note_comments -from app.models import AnalysisStatus -from app.models import CompromiseStatus -from app.models import TaskAssignee -from app.models import AssetsType -from app.models import CaseAssets -from app.models import CaseEventsAssets -from app.models import CaseEventsIoc -from app.models import CaseReceivedFile -from app.models import CaseTasks -from app.models import Cases -from app.models import CasesEvent -from app.models import Comments -from app.models import EventCategory -from app.models import Ioc -from app.models import IocAssetLink -from app.models import IocType -from app.models import Notes -from app.models import NotesGroup -from app.models import TaskStatus -from app.models import Tlp +from app.models.models import AnalysisStatus +from app.models.models import CompromiseStatus +from app.models.models import TaskAssignee +from app.models.models import AssetsType +from app.models.models import CaseAssets +from app.models.models import CaseEventsAssets +from app.models.models import CaseEventsIoc +from app.models.models import CaseReceivedFile +from app.models.models import CaseTasks +from app.models.cases import Cases +from app.models.cases import CasesEvent +from app.models.models import Comments +from app.models.models import EventCategory +from app.models.models import Ioc +from app.models.models import IocAssetLink +from app.models.models import IocType +from app.models.models import Notes +from app.models.models import NotesGroup +from app.models.models import TaskStatus +from app.models.models import Tlp from app.models.authorization import User from app.schema.marshables import CaseDetailsSchema from app.schema.marshables import CommentSchema diff --git a/source/app/datamgmt/states.py b/source/app/datamgmt/states.py index d7eefce8a..e182b3a54 100644 --- a/source/app/datamgmt/states.py +++ b/source/app/datamgmt/states.py @@ -21,7 +21,7 @@ from sqlalchemy import and_ from app import db -from app.models import ObjectState +from app.models.models import ObjectState def _update_object_state(object_name, caseid, userid=None) -> ObjectState: diff --git a/source/app/flask_dropzone/__init__.py b/source/app/flask_dropzone/__init__.py index 55301d451..638cf851e 100644 --- a/source/app/flask_dropzone/__init__.py +++ b/source/app/flask_dropzone/__init__.py @@ -15,7 +15,6 @@ from flask import url_for from app.flask_dropzone.utils import get_url -from app.flask_dropzone.utils import random_filename #: defined normal file type allowed_file_extensions = { diff --git a/source/app/iris_engine/access_control/utils.py b/source/app/iris_engine/access_control/utils.py index 792f53649..a715663e4 100644 --- a/source/app/iris_engine/access_control/utils.py +++ b/source/app/iris_engine/access_control/utils.py @@ -5,20 +5,17 @@ import app from app import db from app.datamgmt.manage.manage_access_control_db import check_ua_case_client -from app.models import Cases -from app.models import Client +from app.models.cases import Cases +from app.models.models import Client from app.models.authorization import CaseAccessLevel from app.models.authorization import UserClient from app.models.authorization import Group from app.models.authorization import GroupCaseAccess -from app.models.authorization import Organisation -from app.models.authorization import OrganisationCaseAccess from app.models.authorization import Permissions from app.models.authorization import User from app.models.authorization import UserCaseAccess from app.models.authorization import UserCaseEffectiveAccess from app.models.authorization import UserGroup -from app.models.authorization import UserOrganisation log = app.app.logger @@ -880,192 +877,6 @@ def ac_trace_user_effective_cases_access_2(user_id): return effective_cases_access -def ac_trace_case_access(case_id): - - case = Cases.query.with_entities( - Cases.case_id, - Cases.name - ).filter( - Cases.case_id == case_id - ).first() - - if not case: - return {} - - ocas = OrganisationCaseAccess.query.with_entities( - Organisation.org_name, - Organisation.org_id, - Organisation.org_uuid, - OrganisationCaseAccess.access_level, - User.id.label('user_id'), - User.name.label('user_name'), - User.email.label('user_email'), - User.uuid.label('user_uuid') - ).filter( - and_(OrganisationCaseAccess.case_id == case.case_id, - OrganisationCaseAccess.org_id == UserOrganisation.org_id) - ).join( - OrganisationCaseAccess.org, - UserOrganisation.user - ).all() - - gcas = GroupCaseAccess.query.with_entities( - Group.group_name, - Group.group_id, - Group.group_uuid, - GroupCaseAccess.access_level, - User.id.label('user_id'), - User.name.label('user_name'), - User.email.label('user_email'), - User.uuid.label('user_uuid') - ).filter( - and_(GroupCaseAccess.case_id == case.case_id, - UserGroup.group_id == GroupCaseAccess.group_id) - ).join( - GroupCaseAccess.group, - UserGroup.user - ).all() - - ucas = UserCaseAccess.query.with_entities( - User.id.label('user_id'), - User.name.label('user_name'), - User.uuid.label('user_uuid'), - User.email.label('user_email'), - UserCaseAccess.access_level - ).filter( - and_(UserCaseAccess.case_id == case.case_id) - ).join( - UserCaseAccess.user - ).all() - - case_access = {} - - for uca in ucas: - user = { - 'access_trace': [], - 'user_effective_access': 0, - 'user_effective_access_list': [], - 'user_info': { - 'user_name': uca.user_name, - 'user_uuid': uca.user_uuid, - 'user_email': uca.user_email - } - } - for ac_l in CaseAccessLevel: - - if uca: - if ac_flag_match_mask(uca.access_level, ac_l.value): - user['user_effective_access'] |= uca.access_level - user['access_trace'].append({ - 'state': 'Effective', - 'name': ac_l.name, - 'value': ac_l.value, - 'inherited_from': { - 'object_type': 'user_access_level', - 'object_name': 'self', - 'object_id': 'self', - 'object_uuid': 'self' - } - }) - user['user_effective_access_list'].append(ac_l.name) - has_uca_overwritten = True - if ac_l.value == CaseAccessLevel.deny_all.value: - has_uca_deny_all = True - - if uca.user_id not in case_access: - case_access.update({ - uca.user_id: user - }) - - for gca in gcas: - if gca.user_id not in case_access: - user = { - 'access_trace': [], - 'user_effective_access': 0, - 'user_effective_access_list': [], - 'user_info': { - 'user_name': gca.user_name, - 'user_uuid': gca.user_uuid, - 'user_email': gca.user_email - } - } - else: - user = case_access[gca.user_id] - - for ac_l in CaseAccessLevel: - - if gca: - if ac_flag_match_mask(gca.access_level, ac_l.value): - if gca.user_id not in case_access: - user['user_effective_access'] |= gca.access_level - user['user_effective_access_list'].append(ac_l.name) - state = 'Effective' - else: - state = 'Overwritten by user access' - - user['access_trace'].append({ - 'state': state, - 'name': ac_l.name, - 'value': ac_l.value, - 'inherited_from': { - 'object_type': 'group_access_level', - 'object_name': gca.group_name, - 'object_id': gca.group_id, - 'object_uuid': gca.group_uuid - } - }) - - if gca.user_id not in case_access: - case_access.update({ - gca.user_id: user - }) - - for oca in ocas: - if oca.user_id not in case_access: - user = { - 'access_trace': [], - 'user_effective_access': 0, - 'user_effective_access_list': [], - 'user_info': { - 'user_name': oca.user_name, - 'user_uuid': oca.user_uuid, - 'user_email': oca.user_email - } - } - else: - user = case_access[oca.user_id] - - for ac_l in CaseAccessLevel: - - if oca: - if ac_flag_match_mask(oca.access_level, ac_l.value): - if oca.user_id not in case_access: - user['user_effective_access'] |= oca.access_level - user['user_effective_access_list'].append(ac_l.name) - state = 'Effective' - else: - state = 'Overwritten by user or group access' - - user['access_trace'].append({ - 'state': state, - 'name': ac_l.name, - 'value': ac_l.value, - 'inherited_from': { - 'object_type': 'organisation_access_level', - 'object_name': oca.org_name, - 'object_id': oca.org_id, - 'object_uuid': oca.org_uuid - } - }) - - if oca.user_id not in case_access: - case_access.update({ - oca.user_id: user - }) - - return case_access - - def ac_get_mask_case_access_level_full(): """ Return a mask for full access level diff --git a/source/app/iris_engine/demo_builder.py b/source/app/iris_engine/demo_builder.py index 3151badbc..57b67f83c 100644 --- a/source/app/iris_engine/demo_builder.py +++ b/source/app/iris_engine/demo_builder.py @@ -28,9 +28,9 @@ from app.datamgmt.manage.manage_users_db import add_user_to_organisation from app.datamgmt.manage.manage_users_db import user_exists from app.iris_engine.access_control.utils import ac_add_users_multi_effective_access -from app.models import Cases -from app.models import Client -from app.models import get_or_create +from app.models.cases import Cases +from app.models.models import Client +from app.models.models import get_or_create from app.models.authorization import CaseAccessLevel from app.models.authorization import User diff --git a/source/app/iris_engine/module_handler/module_handler.py b/source/app/iris_engine/module_handler/module_handler.py index ad3aac0d2..9011d94ef 100644 --- a/source/app/iris_engine/module_handler/module_handler.py +++ b/source/app/iris_engine/module_handler/module_handler.py @@ -34,9 +34,9 @@ from app.datamgmt.iris_engine.modules_db import iris_module_add from app.datamgmt.iris_engine.modules_db import iris_module_exists from app.datamgmt.iris_engine.modules_db import modules_list_pipelines -from app.models import IrisHook -from app.models import IrisModule -from app.models import IrisModuleHook +from app.models.models import IrisHook +from app.models.models import IrisModule +from app.models.models import IrisModuleHook from app.util import hmac_sign from app.util import hmac_verify from iris_interface import IrisInterfaceStatus as IStatus diff --git a/source/app/iris_engine/reporter/reporter.py b/source/app/iris_engine/reporter/reporter.py index 0211acefc..c90253f8e 100644 --- a/source/app/iris_engine/reporter/reporter.py +++ b/source/app/iris_engine/reporter/reporter.py @@ -34,14 +34,14 @@ from app.datamgmt.activities.activities_db import get_manual_activities from app.datamgmt.case.case_db import case_get_desc_crc -from app.models import AssetsType -from app.models import CaseAssets -from app.models import CaseEventsAssets -from app.models import CaseReceivedFile -from app.models import CaseTemplateReport -from app.models import CasesEvent -from app.models import Ioc -from app.models import IocAssetLink +from app.models.models import AssetsType +from app.models.models import CaseAssets +from app.models.models import CaseEventsAssets +from app.models.models import CaseReceivedFile +from app.models.models import CaseTemplateReport +from app.models.cases import CasesEvent +from app.models.models import Ioc +from app.models.models import IocAssetLink from app.iris_engine.reporter.ImageHandler import ImageHandler from app.iris_engine.utils.common import IrisJinjaEnv @@ -165,8 +165,6 @@ def get_case_timeline(caseid): CasesEvent.event_date ).all() - cache_id = {} - ras = {} tim = [] for row in timeline: ras = row @@ -410,8 +408,6 @@ def get_case_timeline(caseid): CasesEvent.event_date ).all() - cache_id = {} - ras = {} tim = [] for row in timeline: ras = row @@ -571,31 +567,3 @@ def generate_md_report(self, doc_type): return None, e.__str__() return output_file_path, 'Report generated' - - -class QueuingHandler(log.Handler): - """A thread safe logging.Handler that writes messages into a queue object. - - Designed to work with LoggingWidget so log messages from multiple - threads can be shown together in a single ttk.Frame. - - The standard logging.QueueHandler/logging.QueueListener can not be used - for this because the QueueListener runs in a private thread, not the - main thread. - - Warning: If multiple threads are writing into this Handler, all threads - must be joined before calling logging.shutdown() or any other log - destinations will be corrupted. - """ - - def __init__(self, *args, task_self, message_queue, **kwargs): - """Initialize by copying the queue and sending everything else to superclass.""" - log.Handler.__init__(self, *args, **kwargs) - self.message_queue = message_queue - self.task_self = task_self - - def emit(self, record): - """Add the formatted log message (sans newlines) to the queue.""" - self.message_queue.append(self.format(record).rstrip('\n')) - self.task_self.update_state(state='PROGRESS', - meta=list(self.message_queue)) diff --git a/source/app/iris_engine/updater/updater.py b/source/app/iris_engine/updater/updater.py index 3e9fcb664..680d83fc6 100644 --- a/source/app/iris_engine/updater/updater.py +++ b/source/app/iris_engine/updater/updater.py @@ -39,7 +39,7 @@ from app import socket_io from app.datamgmt.manage.manage_srv_settings_db import get_server_settings_as_dict from app.iris_engine.backup.backup import backup_iris_db -from app.models import ServerSettings +from app.models.models import ServerSettings from iris_interface import IrisInterfaceStatus as IStatus log = app.logger diff --git a/source/app/iris_engine/utils/tracker.py b/source/app/iris_engine/utils/tracker.py index 59e086caf..bdddec88a 100644 --- a/source/app/iris_engine/utils/tracker.py +++ b/source/app/iris_engine/utils/tracker.py @@ -16,19 +16,17 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# IMPORTS ------------------------------------------------ from datetime import datetime from flask import request from flask_login import current_user import app from app import db -from app.models import UserActivity +from app.models.models import UserActivity log = app.app.logger -# CONTENT ------------------------------------------------ def track_activity(message, caseid=None, ctx_less=False, user_input=False, display_in_ui=True): """ Register a user activity in DB. diff --git a/source/app/models/__init__.py b/source/app/models/__init__.py index 2d7adf33b..e69de29bb 100644 --- a/source/app/models/__init__.py +++ b/source/app/models/__init__.py @@ -1,6 +0,0 @@ -from app.models.models import ObjectState -from app.models.cases import Cases -from app.models.cases import CasesEvent -from app.models.cases import Client -from app.models.models import * - diff --git a/source/app/models/alerts.py b/source/app/models/alerts.py index 47ad035df..ed3b21f36 100644 --- a/source/app/models/alerts.py +++ b/source/app/models/alerts.py @@ -2,7 +2,8 @@ import uuid from sqlalchemy.dialects.postgresql import JSON -from sqlalchemy import BigInteger, Table, Boolean, String +from sqlalchemy import BigInteger +from sqlalchemy import String from sqlalchemy import Column from sqlalchemy import DateTime from sqlalchemy import ForeignKey @@ -13,7 +14,8 @@ from sqlalchemy.orm import relationship from app import db -from app.models import alert_assets_association, alert_iocs_association +from app.models.models import alert_assets_association +from app.models.models import alert_iocs_association class AlertCaseAssociation(db.Model): diff --git a/source/app/models/authorization.py b/source/app/models/authorization.py index df19f59ab..2188b2f47 100644 --- a/source/app/models/authorization.py +++ b/source/app/models/authorization.py @@ -2,7 +2,8 @@ import secrets import uuid from flask_login import UserMixin -from sqlalchemy import BigInteger, JSON +from sqlalchemy import BigInteger +from sqlalchemy import JSON from sqlalchemy import Boolean from sqlalchemy import Column from sqlalchemy import ForeignKey diff --git a/source/app/post_init.py b/source/app/post_init.py index c571eb721..29c0abf8e 100644 --- a/source/app/post_init.py +++ b/source/app/post_init.py @@ -28,7 +28,10 @@ import time from alembic import command from alembic.config import Config -from sqlalchemy import create_engine, exc, or_, text +from sqlalchemy import create_engine +from sqlalchemy import exc +from sqlalchemy import or_ +from sqlalchemy import text from sqlalchemy_utils import create_database from sqlalchemy_utils import database_exists @@ -48,15 +51,22 @@ from app.iris_engine.module_handler.module_handler import check_module_health from app.iris_engine.module_handler.module_handler import instantiate_module_from_name from app.iris_engine.module_handler.module_handler import register_module -from app.models import create_safe_limited -from app.models.alerts import Severity, AlertStatus, AlertResolutionStatus +from app.models.models import create_safe_limited +from app.models.alerts import Severity +from app.models.alerts import AlertStatus +from app.models.alerts import AlertResolutionStatus from app.models.authorization import CaseAccessLevel from app.models.authorization import Group from app.models.authorization import Organisation from app.models.authorization import User -from app.models.cases import Cases, CaseState -from app.models.cases import Client -from app.models.models import AnalysisStatus, CaseClassification, ReviewStatus, ReviewStatusList, EvidenceTypes +from app.models.cases import Cases +from app.models.cases import CaseState +from app.models.models import Client +from app.models.models import AnalysisStatus +from app.models.models import CaseClassification +from app.models.models import ReviewStatus +from app.models.models import ReviewStatusList +from app.models.models import EvidenceTypes from app.models.models import AssetsType from app.models.models import EventCategory from app.models.models import IocType diff --git a/source/app/schema/marshables.py b/source/app/schema/marshables.py index 231348e3c..dd354dc8e 100644 --- a/source/app/schema/marshables.py +++ b/source/app/schema/marshables.py @@ -53,37 +53,37 @@ from app.datamgmt.manage.manage_tags_db import add_db_tag from app.datamgmt.case.case_iocs_db import get_ioc_links from app.iris_engine.access_control.utils import ac_mask_from_val_list -from app.models import AnalysisStatus -from app.models import CaseClassification -from app.models import SavedFilter -from app.models import DataStorePath -from app.models import IrisModuleHook -from app.models import Tags -from app.models import ReviewStatus -from app.models import EvidenceTypes -from app.models import CaseStatus -from app.models import NoteDirectory -from app.models import NoteRevisions -from app.models import AssetsType -from app.models import CaseAssets -from app.models import CaseReceivedFile -from app.models import CaseTasks -from app.models import Cases -from app.models import CasesEvent -from app.models import Client -from app.models import Comments -from app.models import Contact -from app.models import DataStoreFile -from app.models import EventCategory -from app.models import GlobalTasks -from app.models import Ioc -from app.models import IocType -from app.models import IrisModule -from app.models import Notes -from app.models import NotesGroup -from app.models import ServerSettings -from app.models import TaskStatus -from app.models import Tlp +from app.models.models import AnalysisStatus +from app.models.models import CaseClassification +from app.models.models import SavedFilter +from app.models.models import DataStorePath +from app.models.models import IrisModuleHook +from app.models.models import Tags +from app.models.models import ReviewStatus +from app.models.models import EvidenceTypes +from app.models.models import CaseStatus +from app.models.models import NoteDirectory +from app.models.models import NoteRevisions +from app.models.models import AssetsType +from app.models.models import CaseAssets +from app.models.models import CaseReceivedFile +from app.models.models import CaseTasks +from app.models.cases import Cases +from app.models.cases import CasesEvent +from app.models.models import Client +from app.models.models import Comments +from app.models.models import Contact +from app.models.models import DataStoreFile +from app.models.models import EventCategory +from app.models.models import GlobalTasks +from app.models.models import Ioc +from app.models.models import IocType +from app.models.models import IrisModule +from app.models.models import Notes +from app.models.models import NotesGroup +from app.models.models import ServerSettings +from app.models.models import TaskStatus +from app.models.models import Tlp from app.models.alerts import Alert from app.models.alerts import Severity from app.models.alerts import AlertStatus diff --git a/source/app/views.py b/source/app/views.py index 10926f909..9888ffeb6 100644 --- a/source/app/views.py +++ b/source/app/views.py @@ -17,8 +17,6 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from app import app -from app import lm from app.blueprints.pages.activities.activities_routes import activities_blueprint from app.blueprints.pages.alerts.alerts_routes import alerts_blueprint from app.blueprints.pages.case.case_routes import case_blueprint @@ -100,104 +98,93 @@ from app.blueprints.rest.v2 import rest_v2_bp from app.models.authorization import User -from app.post_init import run_post_init - - -app.register_blueprint(graphql_blueprint) -app.register_blueprint(dashboard_blueprint) -app.register_blueprint(dashboard_rest_blueprint) -app.register_blueprint(overview_blueprint) -app.register_blueprint(overview_rest_blueprint) -app.register_blueprint(login_blueprint) -app.register_blueprint(profile_blueprint) -app.register_blueprint(profile_rest_blueprint) -app.register_blueprint(search_blueprint) -app.register_blueprint(search_rest_blueprint) -app.register_blueprint(manage_cases_blueprint) -app.register_blueprint(manage_cases_rest_blueprint) -app.register_blueprint(manage_assets_type_blueprint) -app.register_blueprint(manage_assets_type_rest_blueprint) -app.register_blueprint(manage_srv_settings_blueprint) -app.register_blueprint(manage_server_settings_rest_blueprint) -app.register_blueprint(manage_users_blueprint) -app.register_blueprint(manage_users_rest_blueprint) -app.register_blueprint(manage_templates_blueprint) -app.register_blueprint(manage_templates_rest_blueprint) -app.register_blueprint(manage_modules_blueprint) -app.register_blueprint(manage_modules_rest_blueprint) -app.register_blueprint(manage_customers_blueprint) -app.register_blueprint(manage_customers_rest_blueprint) -app.register_blueprint(manage_analysis_status_rest_blueprint) -app.register_blueprint(manage_ioc_type_blueprint) -app.register_blueprint(manage_ioc_type_rest_blueprint) -app.register_blueprint(manage_event_categories_rest_blueprint) -app.register_blueprint(manage_objects_blueprint) -app.register_blueprint(manage_tlp_type_rest_blueprint) -app.register_blueprint(manage_case_templates_blueprint) -app.register_blueprint(manage_case_templates_rest_blueprint) -app.register_blueprint(manage_task_status_rest_blueprint) -app.register_blueprint(manage_attributes_blueprint) -app.register_blueprint(manage_attributes_rest_blueprint) -app.register_blueprint(manage_ac_blueprint) -app.register_blueprint(manage_ac_rest_blueprint) -app.register_blueprint(manage_groups_blueprint) -app.register_blueprint(manage_groups_rest_blueprint) -app.register_blueprint(manage_case_classification_blueprint) -app.register_blueprint(manage_case_classification_rest_blueprint) -app.register_blueprint(manage_alerts_status_rest_blueprint) -app.register_blueprint(manage_severities_rest_blueprint) -app.register_blueprint(manage_case_state_blueprint) -app.register_blueprint(manage_case_state_rest_blueprint) -app.register_blueprint(manage_evidence_types_blueprint) -app.register_blueprint(manage_evidence_types_rest_blueprint) -app.register_blueprint(manage_assets_rest_blueprint) -app.register_blueprint(manage_tags_rest_blueprint) -app.register_blueprint(saved_filters_rest_blueprint) - -app.register_blueprint(context_rest_blueprint) -app.register_blueprint(case_timeline_blueprint) -app.register_blueprint(case_timeline_rest_blueprint) -app.register_blueprint(case_notes_blueprint) -app.register_blueprint(case_notes_rest_blueprint) -app.register_blueprint(case_assets_blueprint) -app.register_blueprint(case_assets_rest_blueprint) -app.register_blueprint(case_ioc_blueprint) -app.register_blueprint(case_ioc_rest_blueprint) -app.register_blueprint(case_rfiles_blueprint) -app.register_blueprint(case_evidences_rest_blueprint) -app.register_blueprint(case_graph_blueprint) -app.register_blueprint(case_graph_rest_blueprint) -app.register_blueprint(case_tasks_blueprint) -app.register_blueprint(case_tasks_rest_blueprint) -app.register_blueprint(case_blueprint) -app.register_blueprint(case_rest_blueprint) -app.register_blueprint(reports_rest_blueprint) -app.register_blueprint(activities_blueprint) -app.register_blueprint(activities_rest_blueprint) -app.register_blueprint(dim_tasks_blueprint) -app.register_blueprint(dim_tasks_rest_blueprint) -app.register_blueprint(datastore_blueprint) -app.register_blueprint(datastore_rest_blueprint) -app.register_blueprint(alerts_blueprint) -app.register_blueprint(alerts_rest_blueprint) - -app.register_blueprint(rest_api_blueprint) -app.register_blueprint(demo_blueprint) - -app.register_blueprint(rest_v2_bp) - - -try: - - run_post_init(development=app.config["DEVELOPMENT"]) - -except Exception as e: - app.logger.exception(f"Post init failed. IRIS not started") - raise e +def register_blueprints(app): + app.register_blueprint(graphql_blueprint) + app.register_blueprint(dashboard_blueprint) + app.register_blueprint(dashboard_rest_blueprint) + app.register_blueprint(overview_blueprint) + app.register_blueprint(overview_rest_blueprint) + app.register_blueprint(login_blueprint) + app.register_blueprint(profile_blueprint) + app.register_blueprint(profile_rest_blueprint) + app.register_blueprint(search_blueprint) + app.register_blueprint(search_rest_blueprint) + app.register_blueprint(manage_cases_blueprint) + app.register_blueprint(manage_cases_rest_blueprint) + app.register_blueprint(manage_assets_type_blueprint) + app.register_blueprint(manage_assets_type_rest_blueprint) + app.register_blueprint(manage_srv_settings_blueprint) + app.register_blueprint(manage_server_settings_rest_blueprint) + app.register_blueprint(manage_users_blueprint) + app.register_blueprint(manage_users_rest_blueprint) + app.register_blueprint(manage_templates_blueprint) + app.register_blueprint(manage_templates_rest_blueprint) + app.register_blueprint(manage_modules_blueprint) + app.register_blueprint(manage_modules_rest_blueprint) + app.register_blueprint(manage_customers_blueprint) + app.register_blueprint(manage_customers_rest_blueprint) + app.register_blueprint(manage_analysis_status_rest_blueprint) + app.register_blueprint(manage_ioc_type_blueprint) + app.register_blueprint(manage_ioc_type_rest_blueprint) + app.register_blueprint(manage_event_categories_rest_blueprint) + app.register_blueprint(manage_objects_blueprint) + app.register_blueprint(manage_tlp_type_rest_blueprint) + app.register_blueprint(manage_case_templates_blueprint) + app.register_blueprint(manage_case_templates_rest_blueprint) + app.register_blueprint(manage_task_status_rest_blueprint) + app.register_blueprint(manage_attributes_blueprint) + app.register_blueprint(manage_attributes_rest_blueprint) + app.register_blueprint(manage_ac_blueprint) + app.register_blueprint(manage_ac_rest_blueprint) + app.register_blueprint(manage_groups_blueprint) + app.register_blueprint(manage_groups_rest_blueprint) + app.register_blueprint(manage_case_classification_blueprint) + app.register_blueprint(manage_case_classification_rest_blueprint) + app.register_blueprint(manage_alerts_status_rest_blueprint) + app.register_blueprint(manage_severities_rest_blueprint) + app.register_blueprint(manage_case_state_blueprint) + app.register_blueprint(manage_case_state_rest_blueprint) + app.register_blueprint(manage_evidence_types_blueprint) + app.register_blueprint(manage_evidence_types_rest_blueprint) + app.register_blueprint(manage_assets_rest_blueprint) + app.register_blueprint(manage_tags_rest_blueprint) + app.register_blueprint(saved_filters_rest_blueprint) + + app.register_blueprint(context_rest_blueprint) + app.register_blueprint(case_timeline_blueprint) + app.register_blueprint(case_timeline_rest_blueprint) + app.register_blueprint(case_notes_blueprint) + app.register_blueprint(case_notes_rest_blueprint) + app.register_blueprint(case_assets_blueprint) + app.register_blueprint(case_assets_rest_blueprint) + app.register_blueprint(case_ioc_blueprint) + app.register_blueprint(case_ioc_rest_blueprint) + app.register_blueprint(case_rfiles_blueprint) + app.register_blueprint(case_evidences_rest_blueprint) + app.register_blueprint(case_graph_blueprint) + app.register_blueprint(case_graph_rest_blueprint) + app.register_blueprint(case_tasks_blueprint) + app.register_blueprint(case_tasks_rest_blueprint) + app.register_blueprint(case_blueprint) + app.register_blueprint(case_rest_blueprint) + app.register_blueprint(reports_rest_blueprint) + app.register_blueprint(activities_blueprint) + app.register_blueprint(activities_rest_blueprint) + app.register_blueprint(dim_tasks_blueprint) + app.register_blueprint(dim_tasks_rest_blueprint) + app.register_blueprint(datastore_blueprint) + app.register_blueprint(datastore_rest_blueprint) + app.register_blueprint(alerts_blueprint) + app.register_blueprint(alerts_rest_blueprint) + + app.register_blueprint(rest_api_blueprint) + app.register_blueprint(demo_blueprint) + + app.register_blueprint(rest_v2_bp) + # provide login manager with load_user callback -@lm.user_loader def load_user(user_id): return User.query.get(int(user_id)) @@ -213,7 +200,6 @@ def _get_user_by_api_key(api_key): ).first() -@lm.request_loader def load_user_from_request(request): api_key_sources = [ request.headers.get('X-IRIS-AUTH'), diff --git a/source/tests/performance/test_burst_db_interaction.py b/source/tests/performance/test_burst_db_interaction.py index d523f3dfd..5ffb348af 100644 --- a/source/tests/performance/test_burst_db_interaction.py +++ b/source/tests/performance/test_burst_db_interaction.py @@ -31,7 +31,7 @@ from app.datamgmt.manage.manage_users_db import create_user from app.models.cases import Cases from app.models.cases import CasesEvent -from app.models.cases import Client +from app.models.models import Client from app.models.models import CaseEventsAssets from app.models.authorization import User from app.post_init import run_post_init @@ -157,8 +157,6 @@ def random_date(start, end): @staticmethod def update_dates(): - d1 = datetime.strptime('1/1/2008 1:30 PM', '%m/%d/%Y %I:%M %p') - d2 = datetime.strptime('12/12/2021 4:50 AM', '%m/%d/%Y %I:%M %p') events = CasesEvent.query.all() for event in events: event.event_date = datetime.utcnow() diff --git a/source/tests/test_helper.py b/source/tests/test_helper.py index 202a66209..e70841ead 100644 --- a/source/tests/test_helper.py +++ b/source/tests/test_helper.py @@ -27,7 +27,7 @@ from app import app from app.datamgmt.client.client_db import create_client -from app.models import Client +from app.models.models import Client class TestHelper(TestCase):