feat: finish study

AtticusZeller · Sep 1, 2024 · 97f31f4 · 97f31f4
1 parent 6f48d50
commit 97f31f4
Show file tree

Hide file tree

Showing 10 changed files with 260 additions and 109 deletions.
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/ruff.xml b/.idea/ruff.xml
diff --git a/.vscode/PythonImportHelper-v2-Completion.json b/.vscode/PythonImportHelper-v2-Completion.json
@@ -2532,11 +2532,84 @@
         "documentation": {}
     },
     {
-        "label": "datetime",
-        "importPath": "datetime",
-        "description": "datetime",
+        "label": "collections",
+        "kind": 6,
         "isExtraImport": true,
-        "detail": "datetime",
+        "importPath": "collections",
+        "description": "collections",
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "defaultdict",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "OrderedDict",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "OrderedDict",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "defaultdict",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "namedtuple",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "namedtuple",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "namedtuple",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "namedtuple",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
+        "documentation": {}
+    },
+    {
+        "label": "namedtuple",
+        "importPath": "collections",
+        "description": "collections",
+        "isExtraImport": true,
+        "detail": "collections",
         "documentation": {}
     },
     {
@@ -2548,11 +2621,11 @@
         "documentation": {}
     },
     {
-        "label": "DataFrame",
-        "importPath": "pandas",
-        "description": "pandas",
+        "label": "datetime",
+        "importPath": "datetime",
+        "description": "datetime",
         "isExtraImport": true,
-        "detail": "pandas",
+        "detail": "datetime",
         "documentation": {}
     },
     {
@@ -4144,79 +4217,6 @@
         "detail": "gaussian_rasterizer",
         "documentation": {}
     },
-    {
-        "label": "collections",
-        "kind": 6,
-        "isExtraImport": true,
-        "importPath": "collections",
-        "description": "collections",
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "OrderedDict",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "OrderedDict",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "defaultdict",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "namedtuple",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "namedtuple",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "namedtuple",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "namedtuple",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
-    {
-        "label": "namedtuple",
-        "importPath": "collections",
-        "description": "collections",
-        "isExtraImport": true,
-        "detail": "collections",
-        "documentation": {}
-    },
     {
         "label": "Evaluator",
         "importPath": "src.evaluation.evaluator",
@@ -7395,7 +7395,7 @@
         "kind": 5,
         "importPath": "src.plot_rmse",
         "description": "src.plot_rmse",
-        "peekOfCode": "ROOT = Path(__file__).parents[1].as_posix()\nsys.path.append(ROOT)\nfrom src.eval.logger import WandbLogger\nif __name__ == \"__main__\":\n    baseline = \"replica_icps\"\n    # baseline = \"tum\"\n    log = WandbLogger(\n        run_name=\"plot_RMSE\", config={\"description\": f\"baseline_{baseline}\"}\n    )\n    log.plot_RMSE(f\"baseline_{baseline}\")",
+        "peekOfCode": "ROOT = Path(__file__).parents[1].as_posix()\nsys.path.append(ROOT)\nfrom src.eval.logger import WandbLogger\nif __name__ == \"__main__\":\n    baseline = \"\"\n    # baseline = \"tum\"\n    log = WandbLogger(\n        run_name=\"plot_RMSE\", config={\"description\": f\"baseline_{baseline}\"}\n    )\n    # log.plot_RMSE(f\"baseline_{baseline}\")",
         "detail": "src.plot_rmse",
         "documentation": {}
     },
@@ -15778,4 +15778,4 @@
         "detail": "thirdparty.small_gicp.src.test.python_test",
         "documentation": {}
     }
-]
+]
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,3 +1,4 @@
 {
-    "DockerRun.DisableAutoGenerateConfig": true
-}
+    "DockerRun.DisableAutoGenerateConfig": true,
+    "editor.cursorSmoothCaretAnimation": "on"
+}
diff --git a/src/eval/logger.py b/src/eval/logger.py
@@ -1,15 +1,18 @@
+import json
+from collections import defaultdict
 from datetime import datetime
+
 from pathlib import Path
 from typing import Literal
 
 import torch
 import wandb
 from matplotlib import pyplot as plt
-from pandas import DataFrame
 from torch import Tensor
 
 from .utils import calculate_RMSE_np, compute_silhouette_diff
 
+
 # os.environ["WANDB_API_KEY"] = "cedd2caf3e18114de5c6bac2c2c789298ece4ea5"
 # os.environ["WANDB_MODE"] = "offline"
 
@@ -248,14 +251,13 @@ def plot_bar(
         labels: list,
         values: list,
     ):
-
         data = [[label, val] for (label, val) in zip(labels, values)]
         table = wandb.Table(data=data, columns=[label_name, value_name])
         wandb.log(
             {plot_name: wandb.plot.bar(table, label_name, value_name, title=plot_name)}
         )
 
-    def load_history(self, tags: str = "gsplatloc") -> dict[DataFrame]:
+    def load_history(self, tags: str = "baseline", mode: str = "overwrite") -> None:
         """
         https://docs.wandb.ai/ref/python/public-api/api#runs
         Parameters
@@ -264,19 +266,44 @@ def load_history(self, tags: str = "gsplatloc") -> dict[DataFrame]:
 
         Returns
         -------
-        histories: dict[str(sub_set),run.history]
+        histories: list[run.history]
         """
         # filter tags runs
         _runs = self.api.runs(
-            path="supavision/ABGICP", filters={"tags": tags}, order="config.+.sub_set"
+            path=f"{self.entity}/{self.project}",
+            filters={"tags": tags},
+            order="config.+.sub_set",
         )
-        histories = {}
-        for _run in _runs:
+
+        organized_data = defaultdict(lambda: defaultdict(dict))
+        # If appending and file exists, load existing data
+        if mode == "overwrite" and Path("./res.json").exists():
+            with open("./res.json", "r") as f:
+                existing_data = json.load(f)
+            organized_data.update(existing_data)
+
+        for i, _run in enumerate(_runs):
+            print(f"loading {i}th history...")
             run_path = Path(*_run.path).as_posix()
             run = self.api.run(path=run_path)
-            histories[run.config["sub_set"]] = run.history(2000)
-        assert len(histories) == len(_runs)
-        return histories
+            his = run.history(2000)
+
+            eT = his["Translation Error"].to_numpy()
+            eR = his["Rotation Error"].to_numpy()
+            dataset = run.config["dataset"]
+            room = run.config["sub_set"]
+            algorithm = run.config["algorithm"]
+
+            organized_data[dataset][room][algorithm] = {
+                "ATE": calculate_RMSE_np(eT),
+                "AAE": calculate_RMSE_np(eR),
+            }
+        organized_data = json.loads(json.dumps(organized_data))
+
+        with open("./res.json", "w") as f:
+            json.dump(organized_data, f, indent=4)
+
+        print("Data has been saved to res.json")
 
     def plot_RMSE(self, tags: str = "baseline"):
         histories = self.load_history(tags)

diff --git a/src/grip_o3d_finished_experiments.json b/src/grip_o3d_finished_experiments.json
@@ -23,3 +23,4 @@
 ["freiburg3_long_office_household", "PLANE_ICP", "open3d"]
 ["freiburg3_long_office_household", "ICP", "open3d"]
 ["freiburg3_long_office_household", "HYBRID", "open3d"]
+["office3", "HYBRID", "open3d"]
diff --git a/src/icps_eval.py b/src/icps_eval.py
@@ -34,17 +34,19 @@ def save_finished_experiment(file_path: Path, finished: tuple):
         "HYBRID",
     ]
     implements = ("open3d",)
-    rooms = [
-        # "freiburg1_desk",
-        # "freiburg1_desk2",
-        # "freiburg1_room",
-        "freiburg2_xyz",
-        # "freiburg3_long_office_household",
-    ]
-    scenes = "TUM"
+    # rooms = [
+    #     # "freiburg1_desk",
+    #     # "freiburg1_desk2",
+    #     # "freiburg1_room",
+    #     # "freiburg2_xyz",
+    #     # "freiburg3_long_office_household",
+    # ]
+    # scenes = "TUM"
+    scenes = "Replica"
     # rooms = ["room" + str(i) for i in range(3)]
 
     # rooms = ["office" + str(i) for i in range(0, 3)]
+    rooms = ["office" + str(i) for i in range(3, 4)]
     # rooms = ["office" + str(i) for i in range(3, 5)]
 
     finished = load_finished_experiments(file_path)

diff --git a/src/plot_rmse.py b/src/plot_rmse.py
@@ -6,10 +6,13 @@
 from src.eval.logger import WandbLogger
 
 if __name__ == "__main__":
-    baseline = "replica_icps"
+    baseline = ""
     # baseline = "tum"
     log = WandbLogger(
         run_name="plot_RMSE", config={"description": f"baseline_{baseline}"}
     )
 
-    log.plot_RMSE(f"baseline_{baseline}")
+    # log.plot_RMSE(f"baseline_{baseline}")
+    # log.load_history()
+    log.load_history()
+