Ruff step by step

.github/workflows/ci.yml

@@ -27,6 +27,11 @@ jobs:
     steps:
       - name: Check out iris
         uses: actions/checkout@v4
+      - name: Check code with ruff
+        uses: astral-sh/ruff-action@v2
+        with:
+          args: check --output-format=github
+          src: ./source
       - name: Build dockers
         run: |
           # TODO using the environment file from tests to build here. 

e2e/tests/administrator/alerts.spec.js

@@ -17,5 +17,6 @@ test('should present the alert', async ({ page, rest }) => {
             alert_customer_id: 1
         }
     });
+    await page.getByRole('button', { name: 'Refresh', exact: true }).click();
     await expect(page.getByRole('heading', { name: alertTitle })).toBeVisible();
 });

e2e/tests/administrator/case/ioc.spec.js

@@ -1,11 +1,14 @@
 import { test } from '../../restFixture.js';
 import { expect } from '@playwright/test';
+import Api from '../../api.js';
 import crypto from 'node:crypto';
 
 test.beforeEach(async({ page }) => {
     await page.goto('/case/ioc?cid=1');
 });
 
+// TODO should maybe remove all iocs between each tests: there is a risk we reach the pagination limit
+
 test('should be able to update IOC', async ({ page }) => {
     const iocValue = `IOC value - ${crypto.randomUUID()}`;
 
@@ -43,18 +46,7 @@ test('should not be able to create an IOC with the same type and value', async (
 });
 
 test('should paginate the IOCs', async ({ page, rest }) => {
-    const caseName = `Case - ${crypto.randomUUID()}`;
-
-    // TODO maybe should remove cases between each tests (like in the backend tests)
-    let response = await rest.post('/api/v2/cases', {
-        data: {
-            case_name: caseName,
-            case_description: 'Case description',
-            case_customer: 1,
-            case_soc_id: ''
-        }
-    });
-    const caseIdentifier = (await response.json()).case_id;
+    const caseIdentifier = await Api.createCase(rest);
     for (let i = 0; i < 11; i++) {
         await rest.post(`/api/v2/cases/${caseIdentifier}/iocs`, {
             data: {

e2e/tests/administrator/dim/tasks.spec.js

@@ -0,0 +1,39 @@
+import { test } from '../../restFixture.js';
+import { expect } from '@playwright/test';
+import Api from '../../api.js';
+import crypto from 'node:crypto';
+
+let api;
+
+test.beforeEach(async ({ page, rest }) => {
+    await page.goto('/dim/tasks');
+});
+
+test('should be able to consult task info', async ({ page, rest, browserName }) => {
+    let response = await rest.get('/manage/modules/list');
+    const modules = (await response.json()).data;
+    const irisCheckModule = modules.find(module => module.module_human_name === 'IrisCheck');
+    response = await rest.post(`/manage/modules/enable/${irisCheckModule.id}`);
+    const caseIdentifier = await Api.createCase(rest);
+    await rest.delete(`/api/v2/cases/${caseIdentifier}`);
+
+    await page.goto('/dim/tasks');
+    // filter 'Case' column with the case identifier
+    // TODO should make the more interface testable to be able to use some page.getByRole
+    await page.locator('th:nth-child(4) > .form-group > .form-control').fill(caseIdentifier.toString());
+    // filter 'Processing module' column with on_postload_case_delete
+    // TODO should make the more interface testable to be able to use some page.getByRole
+    await page.locator('th:nth-child(5) > .form-group > .form-control').fill('on_postload_case_delete');
+    // TODO should make the more interface testable to be able to use some page.getByRole
+    await page.locator('td').getByRole('link').click();
+    if (browserName === 'chromium') {
+        // TODO this click should not be necessary. However, when run on the chromium browser, it seem the first click is ignored
+        //      => there is probably a bug in the code to chase
+        await page.locator('td').getByRole('link').click();
+    }
+
+    await expect(page.locator('#info_dim_task_modal_body')).toContainText('Module name: iris_check_module');
+    await expect(page.locator('#info_dim_task_modal_body')).toContainText('Hook name: on_postload_case_delete');
+    await expect(page.locator('#info_dim_task_modal_body')).toContainText('User: administrator');
+    await expect(page.locator('#info_dim_task_modal_body')).toContainText(`Case ID: ${caseIdentifier}`);
+});

e2e/tests/api.js

@@ -0,0 +1,20 @@
+import crypto from 'node:crypto';
+
+const createCase = async (rest) => {
+    const caseName = `Case - ${crypto.randomUUID()}`;
+
+    // TODO maybe should remove cases between each tests (like in the backend tests)
+    const response = await rest.post('/api/v2/cases', {
+        data: {
+            case_name: caseName,
+            case_description: 'Case description',
+            case_customer: 1,
+            case_soc_id: ''
+        }
+    });
+    return (await response.json()).case_id;
+};
+
+export default {
+    createCase
+}

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.ruff.lint]
+ignore = ["E402", "E711", "E712", "E721", "E722", "F821"]

source/app/__init__.py

@@ -26,7 +26,8 @@
 from flask_caching import Cache
 from flask_login import LoginManager
 from flask_marshmallow import Marshmallow
-from flask_socketio import SocketIO, Namespace
+from flask_socketio import SocketIO
+from flask_socketio import Namespace
 from flask_sqlalchemy import SQLAlchemy
 from functools import partial
 
@@ -139,4 +140,21 @@ def shutdown_session(exception=None):
     db.session.remove()
 
 
-from app import views
+from app.views import register_blueprints
+from app.views import load_user
+from app.views import load_user_from_request
+
+register_blueprints(app)
+
+from app.post_init import run_post_init
+
+try:
+
+    run_post_init(development=app.config['DEVELOPMENT'])
+
+except Exception as e:
+    app.logger.exception('Post init failed. IRIS not started')
+    raise e
+
+lm.user_loader(load_user)
+lm.request_loader(load_user_from_request)

source/app/alembic/versions/4ecdfcb34f7c_add_compromise_status_to_assets.py

@@ -10,7 +10,7 @@
 
 # revision identifiers, used by Alembic.
 from app.alembic.alembic_utils import _table_has_column
-from app.models import CompromiseStatus
+from app.models.models import CompromiseStatus
 
 revision = '4ecdfcb34f7c'
 down_revision = 'a929ef458490'

source/app/blueprints/access_controls.py

@@ -48,7 +48,7 @@
 from app.iris_engine.access_control.utils import ac_fast_check_user_has_case_access
 from app.iris_engine.access_control.utils import ac_get_effective_permissions_of_user
 from app.iris_engine.utils.tracker import track_activity
-from app.models import Cases
+from app.models.cases import Cases
 from app.models.authorization import Permissions
 from app.models.authorization import CaseAccessLevel
 

source/app/blueprints/pages/case/case_routes.py

@@ -32,7 +32,7 @@
 from app.forms import PipelinesCaseForm
 from app.iris_engine.access_control.utils import ac_get_all_access_level
 from app.iris_engine.module_handler.module_handler import list_available_pipelines
-from app.models import CaseStatus
+from app.models.models import CaseStatus
 from app.models.authorization import CaseAccessLevel
 from app.blueprints.access_controls import ac_case_requires
 

source/app/blueprints/pages/dim_tasks/dim_tasks.py

@@ -26,9 +26,10 @@
 import app
 from app.models.authorization import CaseAccessLevel
 from app.models.authorization import Permissions
-from app.blueprints.access_controls import ac_case_requires, ac_requires
+from app.blueprints.access_controls import ac_case_requires
+from app.blueprints.access_controls import ac_requires
 from app.blueprints.responses import response_error
-from iris_interface.IrisInterfaceStatus import IIStatus
+from app.business.dim_tasks import dim_tasks_get
 
 dim_tasks_blueprint = Blueprint(
     'dim_tasks',
@@ -54,49 +55,7 @@ def dim_index(caseid: int, url_redir):
 @ac_case_requires(CaseAccessLevel.read_only, CaseAccessLevel.full_access)
 def task_status(task_id, caseid, url_redir):
     if url_redir:
-        return response_error("Invalid request")
+        return response_error('Invalid request')
 
-    task = app.celery.AsyncResult(task_id)
-
-    try:
-        tinfo = task.info
-    except AttributeError:
-        # Legacy task
-        task_info = {
-            'Danger': 'This task was executed in a previous version of IRIS and the status cannot be read anymore.',
-            'Note': 'All the data readable by the current IRIS version is displayed in the table.',
-            'Additional information': 'The results of this tasks were stored in a pickled Class which does not exists '
-                                      'anymore in current IRIS version.'
-        }
-        return render_template("modal_task_info.html", data=task_info, task_id=task.id)
-
-    task_info = {
-        'Task ID': task_id,
-        'Task finished on': task.date_done,
-        'Task state': task.state.lower(),
-        'Engine': task.name if task.name else "No engine. Unrecoverable shadow failure"}
-
-    task_meta = task._get_task_meta()
-
-    if task_meta.get('name') \
-            and ('task_hook_wrapper' in task_meta.get('name') or 'pipeline_dispatcher' in task_meta.get('name')):
-        task_info['Module name'] = task_meta.get('kwargs').get('module_name')
-        task_info['Hook name'] = task_meta.get('kwargs').get('hook_name')
-        task_info['User'] = task_meta.get('kwargs').get('init_user')
-        task_info['Case ID'] = task_meta.get('kwargs').get('caseid')
-
-    if isinstance(task.info, IIStatus):
-        success = task.info.is_success()
-        task_info['Logs'] = task.info.get_logs()
-
-    else:
-        success = None
-        task_info['User'] = "Shadow Iris"
-        task_info['Logs'] = ['Task did not returned a valid IIStatus object']
-
-    if task_meta.get('traceback'):
-        task_info['Traceback'] = task.traceback
-
-    task_info['Success'] = "Success" if success else "Failure"
-
-    return render_template("modal_task_info.html", data=task_info, task_id=task.id)
+    task_info = dim_tasks_get(task_id)
+    return render_template('modal_task_info.html', data=task_info)

source/app/blueprints/pages/dim_tasks/templates/modal_task_info.html

@@ -8,7 +8,7 @@
             <label for="en">{{ element }}: </label>
             <span type="text" class="text-faded ml-1" id="en"> {{ data[element] }}</span>
           </div>
-      {% endif %}
+        {% endif %}
       {% endif %}
     {% endfor %}
     </div>

source/app/blueprints/pages/manage/manage_case_templates_routes.py

@@ -22,7 +22,7 @@
 
 from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id
 from app.forms import CaseTemplateForm, AddAssetForm
-from app.models import CaseTemplate
+from app.models.models import CaseTemplate
 from app.models.authorization import Permissions
 from app.blueprints.access_controls import ac_requires
 from app.blueprints.responses import response_error

source/app/blueprints/pages/manage/manage_ioc_types_routes.py

@@ -22,7 +22,7 @@
 from werkzeug.utils import redirect
 
 from app.forms import AddIocTypeForm
-from app.models import IocType
+from app.models.models import IocType
 from app.models.authorization import Permissions
 from app.blueprints.access_controls import ac_requires
 from app.blueprints.responses import response_error

source/app/blueprints/rest/case/case_assets_routes.py

@@ -51,7 +51,7 @@
 from app.iris_engine.access_control.utils import ac_fast_check_current_user_has_case_access
 from app.iris_engine.module_handler.module_handler import call_modules_hook
 from app.iris_engine.utils.tracker import track_activity
-from app.models import AnalysisStatus
+from app.models.models import AnalysisStatus
 from app.models.authorization import CaseAccessLevel
 from app.schema.marshables import CaseAssetsSchema
 from app.schema.marshables import CommentSchema

source/app/blueprints/rest/case/case_notes_routes.py

@@ -46,7 +46,7 @@
 from app.datamgmt.states import get_notes_state
 from app.iris_engine.module_handler.module_handler import call_modules_hook
 from app.iris_engine.utils.tracker import track_activity
-from app.models import Notes
+from app.models.models import Notes
 from app.models.authorization import CaseAccessLevel
 from app.schema.marshables import CaseNoteDirectorySchema
 from app.schema.marshables import CaseNoteRevisionSchema

source/app/blueprints/rest/case/case_routes.py

@@ -43,9 +43,9 @@
 from app.iris_engine.access_control.utils import ac_fast_check_user_has_case_access
 from app.iris_engine.access_control.utils import ac_set_case_access_for_users
 from app.iris_engine.utils.tracker import track_activity
-from app.models import CaseStatus
-from app.models import ReviewStatusList
-from app.models import UserActivity
+from app.models.models import CaseStatus
+from app.models.models import ReviewStatusList
+from app.models.models import UserActivity
 from app.models.authorization import CaseAccessLevel
 from app.models.authorization import User
 from app.schema.marshables import TaskLogSchema

source/app/blueprints/rest/case/case_timeline_routes.py

@@ -54,7 +54,7 @@
 from app.iris_engine.utils.collab import collab_notify
 from app.iris_engine.utils.common import parse_bf_date_format
 from app.iris_engine.utils.tracker import track_activity
-from app.models import CompromiseStatus
+from app.models.models import CompromiseStatus
 from app.models.authorization import CaseAccessLevel
 from app.models.authorization import User
 from app.models.cases import CasesEvent

source/app/blueprints/rest/dim_tasks_routes.py

@@ -23,24 +23,25 @@
 from sqlalchemy import desc
 
 from app.iris_engine.module_handler.module_handler import call_modules_hook
-from app.models import CeleryTaskMeta
-from app.models import IrisHook
-from app.models import IrisModule
-from app.models import IrisModuleHook
-from app.models import CaseAssets
-from app.models import CaseReceivedFile
-from app.models import CaseTasks
-from app.models import Cases
-from app.models import CasesEvent
-from app.models import GlobalTasks
-from app.models import Ioc
-from app.models import Notes
+from app.models.models import CeleryTaskMeta
+from app.models.models import IrisHook
+from app.models.models import IrisModule
+from app.models.models import IrisModuleHook
+from app.models.models import CaseAssets
+from app.models.models import CaseReceivedFile
+from app.models.models import CaseTasks
+from app.models.cases import Cases
+from app.models.cases import CasesEvent
+from app.models.models import GlobalTasks
+from app.models.models import Ioc
+from app.models.models import Notes
 from app.models.alerts import Alert
 from app.models.authorization import CaseAccessLevel
 from app.blueprints.access_controls import ac_requires_case_identifier
 from app.blueprints.access_controls import ac_api_requires
 from app.blueprints.responses import response_error
 from app.blueprints.responses import response_success
+from app.business.dim_tasks import dim_tasks_is_legacy
 from iris_interface.IrisInterfaceStatus import IIStatus
 
 dim_tasks_rest_blueprint = Blueprint('dim_tasks_rest', __name__)
@@ -186,10 +187,7 @@ def list_dim_tasks(count):
 
         tkp = {'state': row.status, 'case': "Unknown", 'module': row.name, 'task_id': row.task_id, 'date_done': row.date_done, 'user': "Unknown"}
 
-        try:
-            _ = row.result
-        except AttributeError:
-            # Legacy task
+        if dim_tasks_is_legacy(row):
             data.append(tkp)
             continue
 

source/app/blueprints/rest/manage/manage_case_templates_routes.py

@@ -27,7 +27,7 @@
 from app.datamgmt.manage.manage_case_templates_db import get_case_template_by_id
 from app.datamgmt.manage.manage_case_templates_db import validate_case_template
 from app.datamgmt.manage.manage_case_templates_db import delete_case_template_by_id
-from app.models import CaseTemplate
+from app.models.models import CaseTemplate
 from app.models.authorization import Permissions
 from app.iris_engine.utils.tracker import track_activity
 from app.schema.marshables import CaseTemplateSchema

source/app/blueprints/rest/manage/manage_ioc_types_routes.py

@@ -24,8 +24,8 @@
 from app.datamgmt.case.case_iocs_db import get_ioc_types_list
 from app.datamgmt.manage.manage_case_objs import search_ioc_type_by_name
 from app.iris_engine.utils.tracker import track_activity
-from app.models import Ioc
-from app.models import IocType
+from app.models.models import Ioc
+from app.models.models import IocType
 from app.models.authorization import Permissions
 from app.schema.marshables import IocTypeSchema
 from app.blueprints.access_controls import ac_api_requires

source/app/blueprints/rest/manage/manage_tlps_routes.py

@@ -18,7 +18,7 @@
 
 from flask import Blueprint
 
-from app.models import Tlp
+from app.models.models import Tlp
 from app.blueprints.access_controls import ac_api_requires
 from app.blueprints.responses import response_error
 from app.blueprints.responses import response_success