test KG construction

idaholab · Dec 5, 2024 · 18b68a0 · 18b68a0
1 parent 6e5dbb4
commit 18b68a0
Show file tree

Hide file tree

Showing 6 changed files with 337 additions and 0 deletions.
diff --git a/examples/KG_demo/KG_Demo_1.ipynb b/examples/KG_demo/KG_Demo_1.ipynb
@@ -0,0 +1,290 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Demo of KG construction for reliability applications\n",
+    "This notebook is intended to show the steps required to construct the KG that include\n",
+    "- MBSE representation of the considered system\n",
+    "- Anomalies detected from numeric data\n",
+    "- Textual elements processed by TLP methods \n",
+    "\n",
+    "This notebook requires that:\n",
+    "* Neo4j desktop is installed in your machine\n",
+    "* A neo4j project is initialized with the corresponding Graph DBMS such that the bolt port is available\n",
+    "* the csv files are located in the correspoding folder; on MAC this can be found in this location:\n",
+    "  /\"User\"/Library/Application Support/Neo4j Desktop/Application/relate-data/dbmss/\"dbms-ID\"/import"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 73,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Source: https://github.com/cj2001/bite_sized_data_science/tree/main\n",
+    "from neo4j import GraphDatabase"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Neo4jConnection:\n",
+    "    \n",
+    "    def __init__(self, uri, user, pwd):\n",
+    "        \n",
+    "        self.__uri = uri\n",
+    "        self.__user = user\n",
+    "        self.__pwd = pwd\n",
+    "        self.__driver = None\n",
+    "        \n",
+    "        try:\n",
+    "            self.__driver = GraphDatabase.driver(self.__uri, auth=(self.__user, self.__pwd))\n",
+    "        except Exception as e:\n",
+    "            print(\"Failed to create the driver:\", e)\n",
+    "        \n",
+    "    def close(self):\n",
+    "        \n",
+    "        if self.__driver is not None:\n",
+    "            self.__driver.close()\n",
+    "        \n",
+    "    def query(self, query, parameters=None, db=None):\n",
+    "        \n",
+    "        assert self.__driver is not None, \"Driver not initialized!\"\n",
+    "        session = None\n",
+    "        response = None\n",
+    "        \n",
+    "        try: \n",
+    "            session = self.__driver.session(database=db) if db is not None else self.__driver.session() \n",
+    "            response = list(session.run(query, parameters))\n",
+    "        except Exception as e:\n",
+    "            print(\"Query failed:\", e)\n",
+    "        finally: \n",
+    "            if session is not None:\n",
+    "                session.close()\n",
+    "        return response\n",
+    "    \n",
+    "uri = \"to be added\"\n",
+    "pwd = \"to be added\"\n",
+    "\n",
+    "conn = Neo4jConnection(uri=uri, user='neo4j', pwd=pwd)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Clean all"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = conn.query('MATCH (n) DETACH DELETE n;')\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Load MBSE nodes and edges"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[<Record row={'nodeId': '0', 'ID': 'C3', 'label': 'C3', 'type': 'entity_emb'}>, <Record row={'nodeId': '1', 'ID': 'cond1', 'label': 'condenser', 'type': 'entity'}>, <Record row={'nodeId': '2', 'ID': 'I_1PC9PKV4BTGV5_BMXNGD2MYES4D', 'label': 'pipe', 'type': 'LML_link'}>, <Record row={'nodeId': '3', 'ID': 'I_9YG45MX0QMK91_80WEVYKF2ZK18', 'label': 'pipe', 'type': 'LML_link'}>, <Record row={'nodeId': '4', 'ID': 'V2', 'label': 'V2', 'type': 'entity_emb'}>, <Record row={'nodeId': '5', 'ID': 'V3', 'label': 'V3', 'type': 'entity_emb'}>, <Record row={'nodeId': '6', 'ID': 'S3', 'label': 'level sensor', 'type': 'entity_emb'}>, <Record row={'nodeId': '7', 'ID': 'None', 'label': 'forebay', 'type': 'entity'}>, <Record row={'nodeId': '8', 'ID': 'body', 'label': 'OPM_pump', 'type': 'MBSE_linked_ent'}>, <Record row={'nodeId': '9', 'ID': 'PM1', 'label': 'pump', 'type': 'entity'}>, <Record row={'nodeId': '10', 'ID': 'I_2J6B8DXJ2WH0N_AR6M5HTA4X3SZ', 'label': 'pipe', 'type': 'LML_link'}>, <Record row={'nodeId': '11', 'ID': 'V1', 'label': 'V1', 'type': 'entity_emb'}>, <Record row={'nodeId': '12', 'ID': 'PM2', 'label': 'pump', 'type': 'entity'}>, <Record row={'nodeId': '13', 'ID': 'I_2G6275A7X6HT5_86DMC9T834MZG', 'label': 'pipe', 'type': 'LML_link'}>]\n",
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "query2 = \"\"\"\n",
+    "LOAD CSV WITH HEADERS FROM 'file:////test_nodes.csv' AS row \n",
+    "MERGE (e:MBSE {nodeId: row.nodeId, label: row.label, ID: row.ID, type: row.type}) return row \"\"\"\n",
+    "\n",
+    "result = conn.query(query2)\n",
+    "print(result)\n",
+    "\n",
+    "query3 = \"\"\"\n",
+    "LOAD CSV WITH HEADERS FROM 'file:///test_edges.csv' AS row\n",
+    "MATCH (e:MBSE {nodeId: row.sourceNodeId})\n",
+    "MATCH (c:MBSE {nodeId: row.targetNodeId})\n",
+    "CREATE (e)-[:MBSE_link {prop:row.type}]->(c) \"\"\"\n",
+    "\n",
+    "result = conn.query(query3)\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load monitoring vars and their link to MBSE nodes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[<Record row={'varID': 'x_1', 'MBSE_ID': '9'}>, <Record row={'varID': 'x_2', 'MBSE_ID': '12'}>, <Record row={'varID': 'T_cond', 'MBSE_ID': '1'}>]\n",
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "query4 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_monit_vars.csv' AS row\n",
+    "MERGE (v:monitor_var {nodeId: row.varID})\n",
+    "return row \"\"\"\n",
+    "result = conn.query(query4)\n",
+    "print(result)\n",
+    "\n",
+    "query5 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_monit_vars.csv' AS row\n",
+    "MATCH (v:monitor_var {nodeId: row.varID})\n",
+    "MATCH (e:MBSE {nodeId: row.MBSE_ID})\n",
+    "CREATE (v)-[:monitoring]->(e) \"\"\"\n",
+    "result = conn.query(query5)\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load anomaly detection methods and their link to monitoring vars"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 78,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[<Record row={'type': 'matrix_profile', 'var_ID': 'x_1', 'AD_ID': '1'}>, <Record row={'type': 'matrix_profile', 'var_ID': 'x_2', 'AD_ID': '2'}>, <Record row={'type': 'matrix_profile', 'var_ID': 'T_cond', 'AD_ID': '3'}>, <Record row={'type': 'matrix_profile_ND', 'var_ID': 'x_1', 'AD_ID': '4'}>, <Record row={'type': 'matrix_profile_ND', 'var_ID': 'x_2', 'AD_ID': '4'}>]\n",
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "query6 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_AD.csv' AS row\n",
+    "MERGE (ad:anomaly_detect {nodeId: row.AD_ID, type: row.type})\n",
+    "return row\"\"\"\n",
+    "result = conn.query(query6)\n",
+    "print(result)\n",
+    "\n",
+    "query7 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_AD.csv' AS row\n",
+    "MATCH (ad:anomaly_detect {nodeId: row.AD_ID})\n",
+    "MATCH (v:monitor_var {nodeId: row.var_ID})\n",
+    "CREATE (ad)-[:input_from]->(v)\"\"\"\n",
+    "result = conn.query(query7)\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load detected anomalies and link to anomaly detection methods"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 79,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[<Record row={'t_in': '1', 't_fin': '2', 'anom_ID': '1', 'AD_ID': '1'}>, <Record row={'t_in': '5.5', 't_fin': '6', 'anom_ID': '2', 'AD_ID': '2'}>, <Record row={'t_in': '8', 't_fin': '9.2', 'anom_ID': '3', 'AD_ID': '3'}>, <Record row={'t_in': '10', 't_fin': '10.5', 'anom_ID': '4', 'AD_ID': '4'}>, <Record row={'t_in': '6.8', 't_fin': '7.1', 'anom_ID': '5', 'AD_ID': '3'}>]\n",
+      "[]\n"
+     ]
+    }
+   ],
+   "source": [
+    "query8 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_anomalies.csv' AS row\n",
+    "MERGE (a:anomaly {nodeId: row.anom_ID, time_initial: row.t_in, time_final: row.t_fin})\n",
+    "return row \"\"\"\n",
+    "result = conn.query(query8)\n",
+    "print(result)\n",
+    "\n",
+    "query9 = \"\"\"LOAD CSV WITH HEADERS FROM 'file:///test_anomalies.csv' AS row\n",
+    "MATCH (a:anomaly {nodeId: row.anom_ID})\n",
+    "MATCH (ad:anomaly_detect {nodeId: row.AD_ID})\n",
+    "CREATE (a)-[:detected_by]->(ad) \"\"\"\n",
+    "result = conn.query(query9)\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[<Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:8' labels=frozenset({'MBSE'}) properties={'label': 'C3', 'ID': 'C3', 'type': 'entity_emb', 'nodeId': '0'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:9' labels=frozenset({'MBSE'}) properties={'ID': 'cond1', 'label': 'condenser', 'type': 'entity', 'nodeId': '1'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:10' labels=frozenset({'MBSE'}) properties={'ID': 'I_1PC9PKV4BTGV5_BMXNGD2MYES4D', 'label': 'pipe', 'type': 'LML_link', 'nodeId': '2'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:11' labels=frozenset({'MBSE'}) properties={'ID': 'I_9YG45MX0QMK91_80WEVYKF2ZK18', 'label': 'pipe', 'type': 'LML_link', 'nodeId': '3'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:12' labels=frozenset({'MBSE'}) properties={'label': 'V2', 'ID': 'V2', 'type': 'entity_emb', 'nodeId': '4'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:13' labels=frozenset({'MBSE'}) properties={'label': 'pump', 'ID': 'PM1', 'type': 'entity', 'nodeId': '9'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:14' labels=frozenset({'MBSE'}) properties={'ID': 'I_2J6B8DXJ2WH0N_AR6M5HTA4X3SZ', 'label': 'pipe', 'type': 'LML_link', 'nodeId': '10'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:15' labels=frozenset({'MBSE'}) properties={'label': 'V1', 'ID': 'V1', 'type': 'entity_emb', 'nodeId': '11'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:16' labels=frozenset({'MBSE'}) properties={'label': 'pump', 'ID': 'PM2', 'type': 'entity', 'nodeId': '12'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:17' labels=frozenset({'MBSE'}) properties={'ID': 'I_2G6275A7X6HT5_86DMC9T834MZG', 'label': 'pipe', 'type': 'LML_link', 'nodeId': '13'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:18' labels=frozenset({'monitor_var'}) properties={'nodeId': 'x_1'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:19' labels=frozenset({'monitor_var'}) properties={'nodeId': 'x_2'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:20' labels=frozenset({'monitor_var'}) properties={'nodeId': 'T_cond'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:21' labels=frozenset({'MBSE'}) properties={'label': 'V3', 'ID': 'V3', 'type': 'entity_emb', 'nodeId': '5'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:22' labels=frozenset({'anomaly_detect'}) properties={'type': 'matrix_profile', 'nodeId': '1'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:23' labels=frozenset({'anomaly_detect'}) properties={'type': 'matrix_profile', 'nodeId': '2'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:24' labels=frozenset({'anomaly_detect'}) properties={'type': 'matrix_profile', 'nodeId': '3'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:25' labels=frozenset({'anomaly_detect'}) properties={'type': 'matrix_profile_ND', 'nodeId': '4'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:26' labels=frozenset({'anomaly'}) properties={'time_initial': '1', 'nodeId': '1', 'time_final': '2'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:27' labels=frozenset({'anomaly'}) properties={'time_initial': '5.5', 'nodeId': '2', 'time_final': '6'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:28' labels=frozenset({'anomaly'}) properties={'time_initial': '8', 'nodeId': '3', 'time_final': '9.2'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:29' labels=frozenset({'anomaly'}) properties={'time_initial': '10', 'nodeId': '4', 'time_final': '10.5'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:30' labels=frozenset({'anomaly'}) properties={'time_initial': '6.8', 'nodeId': '5', 'time_final': '7.1'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:45' labels=frozenset({'MBSE'}) properties={'label': 'level sensor', 'ID': 'S3', 'type': 'entity_emb', 'nodeId': '6'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:46' labels=frozenset({'MBSE'}) properties={'label': 'forebay', 'ID': 'None', 'type': 'entity', 'nodeId': '7'}>>, <Record n=<Node element_id='4:e3d123e1-70ab-499a-8eca-ab1484677aac:47' labels=frozenset({'MBSE'}) properties={'ID': 'body', 'label': 'OPM_pump', 'type': 'MBSE_linked_ent', 'nodeId': '8'}>>]\n"
+     ]
+    }
+   ],
+   "source": [
+    "query10 = \"\"\"MATCH (n) RETURN n \"\"\"\n",
+    "result = conn.query(query10)\n",
+    "print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dackar_libs",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/KG_demo/test_AD.csv b/examples/KG_demo/test_AD.csv
@@ -0,0 +1,6 @@
+AD_ID,var_ID,type
+1,x_1,matrix_profile
+2,x_2,matrix_profile
+3,T_cond,matrix_profile
+4,x_1,matrix_profile_ND
+4,x_2,matrix_profile_ND
diff --git a/examples/KG_demo/test_anomalies.csv b/examples/KG_demo/test_anomalies.csv
@@ -0,0 +1,6 @@
+anom_ID,AD_ID,t_in,t_fin
+1,1,1,2
+2,2,5.5,6
+3,3,8,9.2
+4,4,10,10.5
+5,3,6.8,7.1
diff --git a/examples/KG_demo/test_edges.csv b/examples/KG_demo/test_edges.csv
@@ -0,0 +1,16 @@
+sourceNodeId,targetNodeId,type
+1,0,assoc
+2,9,link
+3,4,assoc
+3,5,assoc
+3,1,link
+7,6,assoc
+7,2,link
+7,13,link
+9,8,MBSElink
+9,10,link
+10,11,assoc
+10,1,link
+12,8,MBSElink
+12,3,link
+13,12,link
diff --git a/examples/KG_demo/test_monit_vars.csv b/examples/KG_demo/test_monit_vars.csv
@@ -0,0 +1,4 @@
+varID,MBSE_ID
+x_1,9
+x_2,12
+T_cond,1
diff --git a/examples/KG_demo/test_nodes.csv b/examples/KG_demo/test_nodes.csv
@@ -0,0 +1,15 @@
+nodeId,label,ID,type
+0,C3,C3,entity_emb
+1,condenser,cond1,entity
+2,pipe,I_1PC9PKV4BTGV5_BMXNGD2MYES4D,LML_link
+3,pipe,I_9YG45MX0QMK91_80WEVYKF2ZK18,LML_link
+4,V2,V2,entity_emb
+5,V3,V3,entity_emb
+6,level sensor,S3,entity_emb
+7,forebay,None,entity
+8,OPM_pump,body,MBSE_linked_ent
+9,pump,PM1,entity
+10,pipe,I_2J6B8DXJ2WH0N_AR6M5HTA4X3SZ,LML_link
+11,V1,V1,entity_emb
+12,pump,PM2,entity
+13,pipe,I_2G6275A7X6HT5_86DMC9T834MZG,LML_link