From 19043c0d20b70eac74a421f109e60efa61b5a8b9 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 15:02:40 -0500
Subject: [PATCH 01/20] =?UTF-8?q?Support=20updating=20`stop`=20documents,?=
 =?UTF-8?q?=20inserting=20when=20needed=20=F0=9F=AB=B7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 018a66e..c5fd041 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -51,6 +51,7 @@ def __init__(self, metadatastore_db, asset_registry_db,
         self._run_start_collection = mds_db.get_collection('run_start')
         self._run_start_collection_revisions = mds_db.get_collection('run_start_revisions')
         self._run_stop_collection = mds_db.get_collection('run_stop')
+        self._run_stop_collection_revisions = mds_db.get_collection('run_stop_revisions')
         self._event_descriptor_collection = mds_db.get_collection(
                                                         'event_descriptor')
         self._event_collection = mds_db.get_collection('event')
@@ -159,9 +160,8 @@ def update(self, name, doc):
         Parameters
         ----------
 
-        name: {'start'}
-            The type of document being updated. Currently, only 'start' is
-            supported and any other value here will raise NotImplementedError.
+        name: {'start'} OR {'stop'} OR {'descriptor'}
+            The type of document being updated.
         doc: dict
             The new version of the document. Its uid will be used to match it
             to the current version, the one to be updated.
@@ -182,6 +182,27 @@ def update(self, name, doc):
             wrapped['document'] = old
             revisions_col.insert_one(wrapped)
             current_col.find_one_and_replace({'uid': doc['uid']}, doc)
+        elif name == 'stop':
+            event_model.schema_validators[event_model.DocumentNames.stop].validate(doc)
+            current_col = self._run_stop_collection
+            revisions_col = self._run_stop_collection_revisions
+            old = current_col.find_one({'run_start': doc['run_start']})
+            if (old is None):
+                # New stop document : insert it
+                current_col.insert_one(doc)
+            else:
+                old.pop('_id')
+                target_uid_docs = revisions_col.find({'document.uid': doc['uid']})
+                cur = target_uid_docs.sort([('revision', pymongo.DESCENDING)]).limit(1)
+                wrapped = dict()
+                try:
+                    wrapped['revision'] = next(cur)['revision'] + 1
+                except StopIteration:
+                    wrapped['revision'] = 0
+                wrapped['document'] = old
+                revisions_col.insert_one(wrapped)
+                current_col.find_one_and_replace({'run_start': doc['run_start']}, doc)
+
         else:
             raise NotImplementedError(
                 f"Updating a {name} document is not currently supported. "

From ea608e7e424c819c3ad08b7d82cef541383df2a6 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 16:50:25 -0500
Subject: [PATCH 02/20] =?UTF-8?q?Reformat=20into=20single,=20more=20versit?=
 =?UTF-8?q?ile=20condition=20=F0=9F=93=A6=EF=B8=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 32 +++++++--------------------
 1 file changed, 8 insertions(+), 24 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index c5fd041..7b676a3 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -166,33 +166,18 @@ def update(self, name, doc):
             The new version of the document. Its uid will be used to match it
             to the current version, the one to be updated.
         """
-        if name == 'start':
+        if name in ['start', 'stop']:
+            key = 'run_start' if name == 'stop' else 'uid'
             event_model.schema_validators[event_model.DocumentNames.start].validate(doc)
-            current_col = self._run_start_collection
-            revisions_col = self._run_start_collection_revisions
-            old = current_col.find_one({'uid': doc['uid']})
-            old.pop('_id')
-            target_uid_docs = revisions_col.find({'document.uid': doc['uid']})
-            cur = target_uid_docs.sort([('revision', pymongo.DESCENDING)]).limit(1)
-            wrapped = dict()
-            try:
-                wrapped['revision'] = next(cur)['revision'] + 1
-            except StopIteration:
-                wrapped['revision'] = 0
-            wrapped['document'] = old
-            revisions_col.insert_one(wrapped)
-            current_col.find_one_and_replace({'uid': doc['uid']}, doc)
-        elif name == 'stop':
-            event_model.schema_validators[event_model.DocumentNames.stop].validate(doc)
-            current_col = self._run_stop_collection
-            revisions_col = self._run_stop_collection_revisions
-            old = current_col.find_one({'run_start': doc['run_start']})
-            if (old is None):
+            current_col = getattr(self, f'_run_{name}_collection')
+            revisions_col = getattr(self, f'_run_{name}_collection_revisions')
+            old = current_col.find_one({key: doc[key]})
+            if (old is None and name == 'stop'):
                 # New stop document : insert it
                 current_col.insert_one(doc)
             else:
                 old.pop('_id')
-                target_uid_docs = revisions_col.find({'document.uid': doc['uid']})
+                target_uid_docs = revisions_col.find({'document.uid': doc[key]})
                 cur = target_uid_docs.sort([('revision', pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()
                 try:
@@ -201,8 +186,7 @@ def update(self, name, doc):
                     wrapped['revision'] = 0
                 wrapped['document'] = old
                 revisions_col.insert_one(wrapped)
-                current_col.find_one_and_replace({'run_start': doc['run_start']}, doc)
-
+                current_col.find_one_and_replace({key : doc[key]}, doc)
         else:
             raise NotImplementedError(
                 f"Updating a {name} document is not currently supported. "

From 8737d3aff94c31bcc0f0be4f791bd5ec9936ff2b Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 17:28:34 -0500
Subject: [PATCH 03/20] =?UTF-8?q?Add=20EXPERIMENTAL=20support=20for=20upda?=
 =?UTF-8?q?ting=20`BlueSkyEventStream`=20descriptors=20=F0=9F=A7=AA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 7b676a3..33e6963 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -166,14 +166,16 @@ def update(self, name, doc):
             The new version of the document. Its uid will be used to match it
             to the current version, the one to be updated.
         """
-        if name in ['start', 'stop']:
-            key = 'run_start' if name == 'stop' else 'uid'
+        if name in ['start', 'stop', 'descriptor']:
+            # Keys and collection names differ slightly between start, stop and descriptor
+            key = 'uid' if name == 'start' else 'run_start'
+            name = f'_event_{name}' if name == 'descriptor' else f'_run_{name}'
             event_model.schema_validators[event_model.DocumentNames.start].validate(doc)
-            current_col = getattr(self, f'_run_{name}_collection')
-            revisions_col = getattr(self, f'_run_{name}_collection_revisions')
+            current_col = getattr(self, f'{name}_collection')
+            revisions_col = getattr(self, f'{name}_collection_revisions')
             old = current_col.find_one({key: doc[key]})
-            if (old is None and name == 'stop'):
-                # New stop document : insert it
+            if (old is None and (name == '_run_stop' or name == '_event_descriptor')):
+                # New stop or descriptor document : insert it
                 current_col.insert_one(doc)
             else:
                 old.pop('_id')

From a522a341cd1c1abbde4ab3106c989b1ec1e22e39 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 17:30:26 -0500
Subject: [PATCH 04/20] =?UTF-8?q?Linter=20formatting=20after=20adding=20hy?=
 =?UTF-8?q?pothetical=20EventStream=20descriptor=20support=20=F0=9F=AB=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 178 +++++++++++++++-----------
 1 file changed, 101 insertions(+), 77 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 33e6963..4e0085b 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -2,14 +2,19 @@
 import pymongo
 from ._version import get_versions
 
-__version__ = get_versions()['version']
+__version__ = get_versions()["version"]
 del get_versions
 
 
 class Serializer(event_model.DocumentRouter):
-    def __init__(self, metadatastore_db, asset_registry_db,
-                 ignore_duplicates=True, resource_uid_unique=False,
-                 tls=False):
+    def __init__(
+        self,
+        metadatastore_db,
+        asset_registry_db,
+        ignore_duplicates=True,
+        resource_uid_unique=False,
+        tls=False,
+    ):
         """
         Insert documents into MongoDB using layout v1.
 
@@ -48,23 +53,28 @@ def __init__(self, metadatastore_db, asset_registry_db,
             assets_db = _get_database(asset_registry_db, tls)
         else:
             assets_db = asset_registry_db
-        self._run_start_collection = mds_db.get_collection('run_start')
-        self._run_start_collection_revisions = mds_db.get_collection('run_start_revisions')
-        self._run_stop_collection = mds_db.get_collection('run_stop')
-        self._run_stop_collection_revisions = mds_db.get_collection('run_stop_revisions')
-        self._event_descriptor_collection = mds_db.get_collection(
-                                                        'event_descriptor')
-        self._event_collection = mds_db.get_collection('event')
-
-        self._resource_collection = assets_db.get_collection('resource')
-        self._datum_collection = assets_db.get_collection('datum')
-
-        self._collections = {'start': self._run_start_collection,
-                             'stop': self._run_stop_collection,
-                             'resource': self._resource_collection,
-                             'descriptor': self._event_descriptor_collection,
-                             'event': self._event_collection,
-                             'datum': self._datum_collection}
+        self._run_start_collection = mds_db.get_collection("run_start")
+        self._run_start_collection_revisions = mds_db.get_collection(
+            "run_start_revisions"
+        )
+        self._run_stop_collection = mds_db.get_collection("run_stop")
+        self._run_stop_collection_revisions = mds_db.get_collection(
+            "run_stop_revisions"
+        )
+        self._event_descriptor_collection = mds_db.get_collection("event_descriptor")
+        self._event_collection = mds_db.get_collection("event")
+
+        self._resource_collection = assets_db.get_collection("resource")
+        self._datum_collection = assets_db.get_collection("datum")
+
+        self._collections = {
+            "start": self._run_start_collection,
+            "stop": self._run_stop_collection,
+            "resource": self._resource_collection,
+            "descriptor": self._event_descriptor_collection,
+            "event": self._event_collection,
+            "datum": self._datum_collection,
+        }
 
         self._metadatastore_db = mds_db
         self._asset_registry_db = assets_db
@@ -78,47 +88,54 @@ def _create_indexes(self):
 
         If the index already exists, this has no effect.
         """
-        self._resource_collection.create_index(
-            'uid', unique=self._resource_uid_unique)
-        self._resource_collection.create_index('resource_id')  # legacy
+        self._resource_collection.create_index("uid", unique=self._resource_uid_unique)
+        self._resource_collection.create_index("resource_id")  # legacy
         # TODO: Migrate all Resources to have a RunStart UID, and then make a
         # unique index on:
         # [('uid', pymongo.ASCENDING), ('run_start', pymongo.ASCENDING)]
-        self._datum_collection.create_index('datum_id', unique=True)
-        self._datum_collection.create_index('resource')
-        self._run_start_collection.create_index('uid', unique=True)
-        self._run_start_collection.create_index('scan_id', unique=False)
+        self._datum_collection.create_index("datum_id", unique=True)
+        self._datum_collection.create_index("resource")
+        self._run_start_collection.create_index("uid", unique=True)
+        self._run_start_collection.create_index("scan_id", unique=False)
         self._run_start_collection.create_index(
-            [('scan_id', pymongo.DESCENDING), ('_id', pymongo.DESCENDING)],
-            unique=True)
+            [("scan_id", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)], unique=True
+        )
         self._run_start_collection.create_index(
-            [('time', pymongo.ASCENDING), ('_id', pymongo.DESCENDING)],
-            unique=True)
+            [("time", pymongo.ASCENDING), ("_id", pymongo.DESCENDING)], unique=True
+        )
         self._run_start_collection.create_index(
-            [('time', pymongo.DESCENDING), ('_id', pymongo.DESCENDING)],
-            unique=True)
+            [("time", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)], unique=True
+        )
         self._run_start_collection.create_index(
-            [('time', pymongo.DESCENDING), ('scan_id', pymongo.DESCENDING)],
-            unique=False, background=True)
+            [("time", pymongo.DESCENDING), ("scan_id", pymongo.DESCENDING)],
+            unique=False,
+            background=True,
+        )
         self._run_start_collection.create_index([("$**", "text")])
-        self._run_start_collection.create_index('data_session', unique=False)
-        self._run_start_collection.create_index('data_groups', unique=False)
-        self._run_stop_collection.create_index('run_start', unique=True)
-        self._run_stop_collection.create_index('uid', unique=True)
+        self._run_start_collection.create_index("data_session", unique=False)
+        self._run_start_collection.create_index("data_groups", unique=False)
+        self._run_stop_collection.create_index("run_start", unique=True)
+        self._run_stop_collection.create_index("uid", unique=True)
         self._run_stop_collection.create_index(
-            [('time', pymongo.DESCENDING)], unique=False, background=True)
+            [("time", pymongo.DESCENDING)], unique=False, background=True
+        )
         self._run_stop_collection.create_index([("$**", "text")])
-        self._event_descriptor_collection.create_index('uid', unique=True)
+        self._event_descriptor_collection.create_index("uid", unique=True)
         self._event_descriptor_collection.create_index(
-            [('run_start', pymongo.DESCENDING), ('time', pymongo.DESCENDING)],
-            unique=False, background=True)
+            [("run_start", pymongo.DESCENDING), ("time", pymongo.DESCENDING)],
+            unique=False,
+            background=True,
+        )
         self._event_descriptor_collection.create_index(
-            [('time', pymongo.DESCENDING)], unique=False, background=True)
+            [("time", pymongo.DESCENDING)], unique=False, background=True
+        )
         self._event_descriptor_collection.create_index([("$**", "text")])
-        self._event_collection.create_index('uid', unique=True)
+        self._event_collection.create_index("uid", unique=True)
         self._event_collection.create_index(
-            [('descriptor', pymongo.DESCENDING), ('time', pymongo.ASCENDING)],
-            unique=False, background=True)
+            [("descriptor", pymongo.DESCENDING), ("time", pymongo.ASCENDING)],
+            unique=False,
+            background=True,
+        )
 
     def __call__(self, name, doc):
         # Before inserting into mongo, convert any numpy objects into built-in
@@ -136,12 +153,14 @@ def _insert(self, name, doc):
                     f"already exists in the database. Document:\n{doc}"
                 ) from err
             else:
-                doc.pop('_id')
+                doc.pop("_id")
                 if name == "datum":
                     id_name = "datum_id"
                 else:
                     id_name = "uid"
-                existing = self._collections[name].find_one({id_name: doc[id_name]}, {'_id': False})
+                existing = self._collections[name].find_one(
+                    {id_name: doc[id_name]}, {"_id": False}
+                )
                 if existing != doc:
                     raise DuplicateUniqueID(
                         "A document with the same unique id as this one "
@@ -166,39 +185,40 @@ def update(self, name, doc):
             The new version of the document. Its uid will be used to match it
             to the current version, the one to be updated.
         """
-        if name in ['start', 'stop', 'descriptor']:
+        if name in ["start", "stop", "descriptor"]:
             # Keys and collection names differ slightly between start, stop and descriptor
-            key = 'uid' if name == 'start' else 'run_start'
-            name = f'_event_{name}' if name == 'descriptor' else f'_run_{name}'
+            key = "uid" if name == "start" else "run_start"
+            name = f"_event_{name}" if name == "descriptor" else f"_run_{name}"
             event_model.schema_validators[event_model.DocumentNames.start].validate(doc)
-            current_col = getattr(self, f'{name}_collection')
-            revisions_col = getattr(self, f'{name}_collection_revisions')
+            current_col = getattr(self, f"{name}_collection")
+            revisions_col = getattr(self, f"{name}_collection_revisions")
             old = current_col.find_one({key: doc[key]})
-            if (old is None and (name == '_run_stop' or name == '_event_descriptor')):
+            if old is None and (name == "_run_stop" or name == "_event_descriptor"):
                 # New stop or descriptor document : insert it
                 current_col.insert_one(doc)
             else:
-                old.pop('_id')
-                target_uid_docs = revisions_col.find({'document.uid': doc[key]})
-                cur = target_uid_docs.sort([('revision', pymongo.DESCENDING)]).limit(1)
+                old.pop("_id")
+                target_uid_docs = revisions_col.find({"document.uid": doc[key]})
+                cur = target_uid_docs.sort([("revision", pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()
                 try:
-                    wrapped['revision'] = next(cur)['revision'] + 1
+                    wrapped["revision"] = next(cur)["revision"] + 1
                 except StopIteration:
-                    wrapped['revision'] = 0
-                wrapped['document'] = old
+                    wrapped["revision"] = 0
+                wrapped["document"] = old
                 revisions_col.insert_one(wrapped)
-                current_col.find_one_and_replace({key : doc[key]}, doc)
+                current_col.find_one_and_replace({key: doc[key]}, doc)
         else:
             raise NotImplementedError(
                 f"Updating a {name} document is not currently supported. "
-                f"Only updates to 'start' documents are supported.")
+                f"Only updates to 'start' documents are supported."
+            )
 
     def start(self, doc):
-        self._insert('start', doc)
+        self._insert("start", doc)
 
     def descriptor(self, doc):
-        self._insert('descriptor', doc)
+        self._insert("descriptor", doc)
 
     def resource(self, doc):
         # In old databases, we know there are duplicates Resources. Until we
@@ -209,9 +229,11 @@ def resource(self, doc):
         # is slow, but since there are never a large number of Resources per
         # Run, this is acceptable.
         if self._resource_uid_unique:
-            self._insert('resource', doc)
+            self._insert("resource", doc)
         else:
-            existing = self._collections["resource"].find_one({'uid': doc['uid']}, {'_id': False})
+            existing = self._collections["resource"].find_one(
+                {"uid": doc["uid"]}, {"_id": False}
+            )
             if existing is not None:
                 if existing != doc:
                     raise DuplicateUniqueID(
@@ -223,7 +245,7 @@ def resource(self, doc):
             self._collections["resource"].insert_one(doc)
 
     def event(self, doc):
-        self._insert('event', doc)
+        self._insert("event", doc)
 
     def event_page(self, doc):
         # Unpack an EventPage into Events and do the actual insert inside
@@ -237,7 +259,7 @@ def event_page(self, doc):
             filled_events.append(event_method(event_doc))
 
     def datum(self, doc):
-        self._insert('datum', doc)
+        self._insert("datum", doc)
 
     def datum_page(self, doc):
         # Unpack an DatumPage into Datum and do the actual insert inside
@@ -251,20 +273,22 @@ def datum_page(self, doc):
             filled_datums.append(datum_method(datum_doc))
 
     def stop(self, doc):
-        self._insert('stop', doc)
+        self._insert("stop", doc)
 
     def __repr__(self):
         # Display connection info in eval-able repr.
-        return (f'{type(self).__name__}('
-                f'metadatastore_db={self._metadatastore_db!r}, '
-                f'asset_registry_db={self._asset_registry_db!r})')
+        return (
+            f"{type(self).__name__}("
+            f"metadatastore_db={self._metadatastore_db!r}, "
+            f"asset_registry_db={self._asset_registry_db!r})"
+        )
 
 
 def _get_database(uri, tls):
-    if not pymongo.uri_parser.parse_uri(uri)['database']:
+    if not pymongo.uri_parser.parse_uri(uri)["database"]:
         raise ValueError(
-            f"Invalid URI: {uri} "
-            f"Did you forget to include a database?")
+            f"Invalid URI: {uri} " f"Did you forget to include a database?"
+        )
     else:
         client = pymongo.MongoClient(uri, tls=tls)
         return client.get_database()

From b3b3aeeb24dca02279471c25d5343541cd81bb12 Mon Sep 17 00:00:00 2001
From: Kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 17:33:55 -0500
Subject: [PATCH 05/20] =?UTF-8?q?Fix=20dynamic=20access=20of=20schema=20va?=
 =?UTF-8?q?lidation=20variables=20=F0=9F=94=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Dan Allan <daniel.b.allan@gmail.com>
---
 suitcase/mongo_normalized/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 4e0085b..60afd74 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -189,7 +189,7 @@ def update(self, name, doc):
             # Keys and collection names differ slightly between start, stop and descriptor
             key = "uid" if name == "start" else "run_start"
             name = f"_event_{name}" if name == "descriptor" else f"_run_{name}"
-            event_model.schema_validators[event_model.DocumentNames.start].validate(doc)
+            event_model.schema_validators[getattr(event_model.DocumentNames, name)].validate(doc)
             current_col = getattr(self, f"{name}_collection")
             revisions_col = getattr(self, f"{name}_collection_revisions")
             old = current_col.find_one({key: doc[key]})

From b86982141e4d50c272f1370baf6b132b7e269119 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Thu, 1 Feb 2024 17:40:22 -0500
Subject: [PATCH 06/20] =?UTF-8?q?Instantiate=20`event=5Fdescriptors=5Frevi?=
 =?UTF-8?q?sions'=20collection=20=E2=9E=95?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 60afd74..c9253fa 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -61,6 +61,9 @@ def __init__(
         self._run_stop_collection_revisions = mds_db.get_collection(
             "run_stop_revisions"
         )
+        self._event_descriptor_collection_revisions = mds_db.get_collection(
+            "event_descriptor_revisions"
+        )
         self._event_descriptor_collection = mds_db.get_collection("event_descriptor")
         self._event_collection = mds_db.get_collection("event")
 

From 413a44eb183f9cfca40da63b8183fa7d5ac5fc2c Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 12:55:25 -0500
Subject: [PATCH 07/20] =?UTF-8?q?Validate=20before=20mutating=20name,=20pa?=
 =?UTF-8?q?ss=20tests=20=E2=9C=85=E2=8F=A9=EF=B8=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index c9253fa..a144751 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -189,10 +189,10 @@ def update(self, name, doc):
             to the current version, the one to be updated.
         """
         if name in ["start", "stop", "descriptor"]:
+            event_model.schema_validators[getattr(event_model.DocumentNames, name)].validate(doc)
             # Keys and collection names differ slightly between start, stop and descriptor
             key = "uid" if name == "start" else "run_start"
             name = f"_event_{name}" if name == "descriptor" else f"_run_{name}"
-            event_model.schema_validators[getattr(event_model.DocumentNames, name)].validate(doc)
             current_col = getattr(self, f"{name}_collection")
             revisions_col = getattr(self, f"{name}_collection_revisions")
             old = current_col.find_one({key: doc[key]})

From 78a256c68d049f9c78924c8f53552bd9bf128dff Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 15:22:10 -0500
Subject: [PATCH 08/20] =?UTF-8?q?Replace=20an=20incredibly=20bad=20hardcod?=
 =?UTF-8?q?ed=20test=20for=20stop=20=E2=8F=B9=EF=B8=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/tests/tests.py | 115 ++++++++++++++---------
 1 file changed, 70 insertions(+), 45 deletions(-)

diff --git a/suitcase/mongo_normalized/tests/tests.py b/suitcase/mongo_normalized/tests/tests.py
index d72612b..8528727 100644
--- a/suitcase/mongo_normalized/tests/tests.py
+++ b/suitcase/mongo_normalized/tests/tests.py
@@ -30,7 +30,7 @@ def test_duplicates(db_factory, example_data):
 
     # Modify a document, check that inserting a document with uid,
     # but different content raises.
-    documents[0][1]['new_key'] = 'new_value'
+    documents[0][1]["new_key"] = "new_value"
     with pytest.raises(DuplicateUniqueID):
         for item in documents:
             serializer(*item)
@@ -43,31 +43,56 @@ def test_update(db_factory, example_data):
     serializer = Serializer(metadatastore_db, asset_registry_db)
     for item in documents:
         serializer(*item)
-    original = documents[0][1]
-    start = copy.deepcopy(original)
-    start['user'] = 'first updated temp user'
-    serializer.update('start', start)
-    real = metadatastore_db.get_collection('run_start').find_one({'uid': start['uid']})
-    real.pop('_id')
-    assert sanitize_doc(real) == sanitize_doc(start)
-    revision = metadatastore_db.get_collection('run_start_revisions').find_one({'document.uid': start['uid']})
-    assert revision['revision'] == 0
-    revision.pop('revision')
-    revision.pop('_id')
-    assert sanitize_doc(revision['document']) == sanitize_doc(original)
+    original_start = next(item[1] for item in documents if item[0] == 'start')
+    original_stop = next(item[1] for item in documents if item[0] == 'stop')
+    original_descriptor = next(item[1] for item in documents if item[0] == 'descriptor')
+    # (1) Make mutable copies
+    start = copy.deepcopy(original_start)
+    stop = copy.deepcopy(original_stop)
+    # (2) Update a property of the copies
+    start["user"] = "first updated temp user"
+    serializer.update("start", start)
+    stop["reason"] = "Everything happens for a reason."
+    serializer.update("stop", stop)
+    # (3) Get the updated record from the database to confirm changes
+    real_start = metadatastore_db.get_collection("run_start").find_one({"uid": start["uid"]})
+    real_start.pop("_id")
+    real_stop = metadatastore_db.get_collection("run_stop").find_one({"run_start": stop["run_start"]})
+    real_stop.pop("_id")
+    # (4) Test the data
+    assert sanitize_doc(real_start) == sanitize_doc(start)
+    assert sanitize_doc(real_stop) == sanitize_doc(stop)
+    # (5) Test the revisions    
+    revision_start = metadatastore_db.get_collection("run_start_revisions").find_one(
+        {"document.uid": start["uid"]}
+    )
+    assert revision_start["revision"] == 0
+    revision_start.pop("revision")
+    revision_start.pop("_id")
+    assert sanitize_doc(revision_start["document"]) == sanitize_doc(original_start)
+
+    revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
+        {"document.run_start": stop["run_start"]}
+    )
+    assert revision_stop["revision"] == 0
+    revision_stop.pop("revision")
+    revision_stop.pop("_id")
+    assert sanitize_doc(revision_stop["document"]) == sanitize_doc(original_stop)
+
 
     revision1 = copy.deepcopy(start)
-    start['user'] = 'second updated temp user'
-    serializer.update('start', start)
-    real = metadatastore_db.get_collection('run_start').find_one({'uid': start['uid']})
-    real.pop('_id')
+    start["user"] = "second updated temp user"
+    serializer.update("start", start)
+    real = metadatastore_db.get_collection("run_start").find_one({"uid": start["uid"]})
+    real.pop("_id")
     assert sanitize_doc(real) == sanitize_doc(start)
-    revision = metadatastore_db.get_collection('run_start_revisions').find_one({'document.uid': start['uid'],
-                                                                                'revision': 1})
-    assert revision['revision'] == 1
-    revision.pop('revision')
-    revision.pop('_id')
-    assert sanitize_doc(revision['document']) == sanitize_doc(revision1)
+    revision = metadatastore_db.get_collection("run_start_revisions").find_one(
+        {"document.uid": start["uid"], "revision": 1}
+    )
+    assert revision["revision"] == 1
+    revision.pop("revision")
+    revision.pop("_id")
+    assert sanitize_doc(revision["document"]) == sanitize_doc(revision1)
 
 
 def test_notimplemented_error(db_factory, example_data):
@@ -75,7 +100,7 @@ def test_notimplemented_error(db_factory, example_data):
     asset_registry_db = db_factory()
     serializer = Serializer(metadatastore_db, asset_registry_db)
     with pytest.raises(NotImplementedError):
-        assert serializer.update('not_start', {})
+        assert serializer.update("not_start", {})
 
 
 def test_validation_error(db_factory, example_data):
@@ -83,7 +108,7 @@ def test_validation_error(db_factory, example_data):
     asset_registry_db = db_factory()
     serializer = Serializer(metadatastore_db, asset_registry_db)
     with pytest.raises(ValidationError):
-        assert serializer.update('start', {})
+        assert serializer.update("start", {})
 
 
 def test_index_creation(db_factory):
@@ -95,40 +120,40 @@ def test_index_creation(db_factory):
 
     indexes = asset_registry_db.resource.index_information()
     assert len(indexes.keys()) == 3
-    assert not indexes['uid_1'].get('unique')
-    assert indexes['resource_id_1']
+    assert not indexes["uid_1"].get("unique")
+    assert indexes["resource_id_1"]
 
     indexes = asset_registry_db.datum.index_information()
     assert len(indexes.keys()) == 3
-    assert indexes['datum_id_1']['unique']
-    assert indexes['resource_1']
+    assert indexes["datum_id_1"]["unique"]
+    assert indexes["resource_1"]
 
     indexes = metadatastore_db.run_start.index_information()
     assert len(indexes.keys()) == 6
-    assert indexes['uid_1']['unique']
-    assert indexes['time_-1_scan_id_-1']
-    assert indexes['$**_text']
-    assert indexes['data_session_1']
-    assert indexes['data_groups_1']
+    assert indexes["uid_1"]["unique"]
+    assert indexes["time_-1_scan_id_-1"]
+    assert indexes["$**_text"]
+    assert indexes["data_session_1"]
+    assert indexes["data_groups_1"]
 
     indexes = metadatastore_db.run_stop.index_information()
     assert len(indexes.keys()) == 5
-    assert indexes['uid_1']['unique']
-    assert indexes['run_start_1']['unique']
-    assert indexes['time_-1']
-    assert indexes['$**_text']
+    assert indexes["uid_1"]["unique"]
+    assert indexes["run_start_1"]["unique"]
+    assert indexes["time_-1"]
+    assert indexes["$**_text"]
 
     indexes = metadatastore_db.event_descriptor.index_information()
     assert len(indexes.keys()) == 5
-    assert indexes['uid_1']['unique']
-    assert indexes['run_start_-1_time_-1']
-    assert indexes['time_-1']
-    assert indexes['$**_text']
+    assert indexes["uid_1"]["unique"]
+    assert indexes["run_start_-1_time_-1"]
+    assert indexes["time_-1"]
+    assert indexes["$**_text"]
 
     indexes = metadatastore_db.event.index_information()
     assert len(indexes.keys()) == 3
-    assert indexes['uid_1']['unique']
-    assert indexes['descriptor_-1_time_1']
+    assert indexes["uid_1"]["unique"]
+    assert indexes["descriptor_-1_time_1"]
 
 
 def test_resource_uid_unique(db_factory):
@@ -139,4 +164,4 @@ def test_resource_uid_unique(db_factory):
     Serializer(metadatastore_db, asset_registry_db, resource_uid_unique=True)
 
     indexes = asset_registry_db.resource.index_information()
-    assert indexes['uid_1'].get('unique')
+    assert indexes["uid_1"].get("unique")

From bb44a883856e1c5c969cb1799d43be16f1cb5889 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 15:48:11 -0500
Subject: [PATCH 09/20] =?UTF-8?q?Version=20is=20not=20incrementing=20in=20?=
 =?UTF-8?q?'stop=5Frevisions'=20collection=20=E2=9D=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/tests/tests.py | 53 +++++++++++++++++-------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/suitcase/mongo_normalized/tests/tests.py b/suitcase/mongo_normalized/tests/tests.py
index 8528727..862a485 100644
--- a/suitcase/mongo_normalized/tests/tests.py
+++ b/suitcase/mongo_normalized/tests/tests.py
@@ -43,9 +43,9 @@ def test_update(db_factory, example_data):
     serializer = Serializer(metadatastore_db, asset_registry_db)
     for item in documents:
         serializer(*item)
-    original_start = next(item[1] for item in documents if item[0] == 'start')
-    original_stop = next(item[1] for item in documents if item[0] == 'stop')
-    original_descriptor = next(item[1] for item in documents if item[0] == 'descriptor')
+    original_start = next(item[1] for item in documents if item[0] == "start")
+    original_stop = next(item[1] for item in documents if item[0] == "stop")
+    original_descriptor = next(item[1] for item in documents if item[0] == "descriptor")
     # (1) Make mutable copies
     start = copy.deepcopy(original_start)
     stop = copy.deepcopy(original_stop)
@@ -55,14 +55,18 @@ def test_update(db_factory, example_data):
     stop["reason"] = "Everything happens for a reason."
     serializer.update("stop", stop)
     # (3) Get the updated record from the database to confirm changes
-    real_start = metadatastore_db.get_collection("run_start").find_one({"uid": start["uid"]})
+    real_start = metadatastore_db.get_collection("run_start").find_one(
+        {"uid": start["uid"]}
+    )
     real_start.pop("_id")
-    real_stop = metadatastore_db.get_collection("run_stop").find_one({"run_start": stop["run_start"]})
+    real_stop = metadatastore_db.get_collection("run_stop").find_one(
+        {"run_start": stop["run_start"]}
+    )
     real_stop.pop("_id")
     # (4) Test the data
     assert sanitize_doc(real_start) == sanitize_doc(start)
     assert sanitize_doc(real_stop) == sanitize_doc(stop)
-    # (5) Test the revisions    
+    # (5) Test the revisions
     revision_start = metadatastore_db.get_collection("run_start_revisions").find_one(
         {"document.uid": start["uid"]}
     )
@@ -79,20 +83,37 @@ def test_update(db_factory, example_data):
     revision_stop.pop("_id")
     assert sanitize_doc(revision_stop["document"]) == sanitize_doc(original_stop)
 
-
-    revision1 = copy.deepcopy(start)
+    # (6) Test another revision
+    revision1_start = copy.deepcopy(start)
+    revision1_stop = copy.deepcopy(stop)
     start["user"] = "second updated temp user"
     serializer.update("start", start)
-    real = metadatastore_db.get_collection("run_start").find_one({"uid": start["uid"]})
-    real.pop("_id")
-    assert sanitize_doc(real) == sanitize_doc(start)
-    revision = metadatastore_db.get_collection("run_start_revisions").find_one(
+    stop["reason"] = "Nothing happens for a reason."
+    serializer.update("stop", stop)
+    real_start = metadatastore_db.get_collection("run_start").find_one(
+        {"uid": start["uid"]}
+    )
+    real_start.pop("_id")
+    assert sanitize_doc(real_start) == sanitize_doc(start)
+    real_stop = metadatastore_db.get_collection("run_stop").find_one(
+        {"run_start": stop["run_start"]}
+    )
+    real_stop.pop("_id")
+    assert sanitize_doc(real_stop) == sanitize_doc(stop)
+    revision_start = metadatastore_db.get_collection("run_start_revisions").find_one(
         {"document.uid": start["uid"], "revision": 1}
     )
-    assert revision["revision"] == 1
-    revision.pop("revision")
-    revision.pop("_id")
-    assert sanitize_doc(revision["document"]) == sanitize_doc(revision1)
+    assert revision_start["revision"] == 1
+    revision_start.pop("revision")
+    revision_start.pop("_id")
+    # revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
+    # {"document.run_start": stop["run_start"],'revision': 1}
+    # )
+    # assert revision_stop["revision"] == 1
+    # revision_stop.pop("revision")
+    # revision_stop.pop("_id")
+    assert sanitize_doc(revision_start["document"]) == sanitize_doc(revision1_start)
+    # assert sanitize_doc(revision_stop["document"]) == sanitize_doc(revision1_stop)
 
 
 def test_notimplemented_error(db_factory, example_data):

From 0a507b5b87b61d8ce14c0551f6f967575052f181 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 15:58:52 -0500
Subject: [PATCH 10/20] =?UTF-8?q?Complete=20tests=20for=20`stop`=20docs,?=
 =?UTF-8?q?=20increment=20`stop`=20revisions=20=F0=9F=94=82?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py    | 11 ++++++-----
 suitcase/mongo_normalized/tests/tests.py | 14 +++++++-------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index a144751..c989863 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -189,7 +189,9 @@ def update(self, name, doc):
             to the current version, the one to be updated.
         """
         if name in ["start", "stop", "descriptor"]:
-            event_model.schema_validators[getattr(event_model.DocumentNames, name)].validate(doc)
+            event_model.schema_validators[
+                getattr(event_model.DocumentNames, name)
+            ].validate(doc)
             # Keys and collection names differ slightly between start, stop and descriptor
             key = "uid" if name == "start" else "run_start"
             name = f"_event_{name}" if name == "descriptor" else f"_run_{name}"
@@ -201,7 +203,7 @@ def update(self, name, doc):
                 current_col.insert_one(doc)
             else:
                 old.pop("_id")
-                target_uid_docs = revisions_col.find({"document.uid": doc[key]})
+                target_uid_docs = revisions_col.find({"document.uid": doc["uid"]})
                 cur = target_uid_docs.sort([("revision", pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()
                 try:
@@ -210,7 +212,7 @@ def update(self, name, doc):
                     wrapped["revision"] = 0
                 wrapped["document"] = old
                 revisions_col.insert_one(wrapped)
-                current_col.find_one_and_replace({key: doc[key]}, doc)
+                current_col.find_one_and_replace({"uid": doc["uid"]}, doc)
         else:
             raise NotImplementedError(
                 f"Updating a {name} document is not currently supported. "
@@ -297,5 +299,4 @@ def _get_database(uri, tls):
         return client.get_database()
 
 
-class DuplicateUniqueID(Exception):
-    ...
+class DuplicateUniqueID(Exception): ...
diff --git a/suitcase/mongo_normalized/tests/tests.py b/suitcase/mongo_normalized/tests/tests.py
index 862a485..dd0bf45 100644
--- a/suitcase/mongo_normalized/tests/tests.py
+++ b/suitcase/mongo_normalized/tests/tests.py
@@ -106,14 +106,14 @@ def test_update(db_factory, example_data):
     assert revision_start["revision"] == 1
     revision_start.pop("revision")
     revision_start.pop("_id")
-    # revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
-    # {"document.run_start": stop["run_start"],'revision': 1}
-    # )
-    # assert revision_stop["revision"] == 1
-    # revision_stop.pop("revision")
-    # revision_stop.pop("_id")
+    revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
+        {"document.run_start": stop["run_start"], "revision": 1}
+    )
+    assert revision_stop["revision"] == 1
+    revision_stop.pop("revision")
+    revision_stop.pop("_id")
     assert sanitize_doc(revision_start["document"]) == sanitize_doc(revision1_start)
-    # assert sanitize_doc(revision_stop["document"]) == sanitize_doc(revision1_stop)
+    assert sanitize_doc(revision_stop["document"]) == sanitize_doc(revision1_stop)
 
 
 def test_notimplemented_error(db_factory, example_data):

From d297ddf8aee50bc29083eb3363ef2bb7726f4e2d Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 16:44:43 -0500
Subject: [PATCH 11/20] =?UTF-8?q?Always=20use=20UID,=20test=20for=20descri?=
 =?UTF-8?q?ptor=20completed=20=F0=9F=94=AC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py    |  2 +-
 suitcase/mongo_normalized/tests/tests.py | 26 ++++++++++++++++++++----
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index c989863..b51ce18 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -193,7 +193,7 @@ def update(self, name, doc):
                 getattr(event_model.DocumentNames, name)
             ].validate(doc)
             # Keys and collection names differ slightly between start, stop and descriptor
-            key = "uid" if name == "start" else "run_start"
+            key = "uid"
             name = f"_event_{name}" if name == "descriptor" else f"_run_{name}"
             current_col = getattr(self, f"{name}_collection")
             revisions_col = getattr(self, f"{name}_collection_revisions")
diff --git a/suitcase/mongo_normalized/tests/tests.py b/suitcase/mongo_normalized/tests/tests.py
index dd0bf45..31a85eb 100644
--- a/suitcase/mongo_normalized/tests/tests.py
+++ b/suitcase/mongo_normalized/tests/tests.py
@@ -49,23 +49,31 @@ def test_update(db_factory, example_data):
     # (1) Make mutable copies
     start = copy.deepcopy(original_start)
     stop = copy.deepcopy(original_stop)
+    descriptor = copy.deepcopy(original_descriptor)
     # (2) Update a property of the copies
     start["user"] = "first updated temp user"
     serializer.update("start", start)
     stop["reason"] = "Everything happens for a reason."
     serializer.update("stop", stop)
+    descriptor["name"] = "secondary"
+    serializer.update("descriptor", descriptor)
     # (3) Get the updated record from the database to confirm changes
     real_start = metadatastore_db.get_collection("run_start").find_one(
         {"uid": start["uid"]}
     )
     real_start.pop("_id")
     real_stop = metadatastore_db.get_collection("run_stop").find_one(
-        {"run_start": stop["run_start"]}
+        {"uid": stop["uid"]}
     )
     real_stop.pop("_id")
+    real_descriptor = metadatastore_db.get_collection("event_descriptor").find_one(
+        {"uid": descriptor["uid"]}
+    )
+    real_descriptor.pop("_id")
     # (4) Test the data
     assert sanitize_doc(real_start) == sanitize_doc(start)
     assert sanitize_doc(real_stop) == sanitize_doc(stop)
+    assert sanitize_doc(real_descriptor) == sanitize_doc(descriptor)
     # (5) Test the revisions
     revision_start = metadatastore_db.get_collection("run_start_revisions").find_one(
         {"document.uid": start["uid"]}
@@ -76,13 +84,23 @@ def test_update(db_factory, example_data):
     assert sanitize_doc(revision_start["document"]) == sanitize_doc(original_start)
 
     revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
-        {"document.run_start": stop["run_start"]}
+        {"document.uid": stop["uid"]}
     )
     assert revision_stop["revision"] == 0
     revision_stop.pop("revision")
     revision_stop.pop("_id")
     assert sanitize_doc(revision_stop["document"]) == sanitize_doc(original_stop)
 
+    revision_descriptor = metadatastore_db.get_collection(
+        "event_descriptor_revisions"
+    ).find_one({"document.uid": descriptor["uid"]})
+    assert revision_descriptor["revision"] == 0
+    revision_descriptor.pop("revision")
+    revision_descriptor.pop("_id")
+    assert sanitize_doc(revision_descriptor["document"]) == sanitize_doc(
+        original_descriptor
+    )
+
     # (6) Test another revision
     revision1_start = copy.deepcopy(start)
     revision1_stop = copy.deepcopy(stop)
@@ -96,7 +114,7 @@ def test_update(db_factory, example_data):
     real_start.pop("_id")
     assert sanitize_doc(real_start) == sanitize_doc(start)
     real_stop = metadatastore_db.get_collection("run_stop").find_one(
-        {"run_start": stop["run_start"]}
+        {"uid": stop["uid"]}
     )
     real_stop.pop("_id")
     assert sanitize_doc(real_stop) == sanitize_doc(stop)
@@ -107,7 +125,7 @@ def test_update(db_factory, example_data):
     revision_start.pop("revision")
     revision_start.pop("_id")
     revision_stop = metadatastore_db.get_collection("run_stop_revisions").find_one(
-        {"document.run_start": stop["run_start"], "revision": 1}
+        {"document.uid": stop["uid"], "revision": 1}
     )
     assert revision_stop["revision"] == 1
     revision_stop.pop("revision")

From 90ff8e3d527c9afca0bcab7dd35e3a784c4d4ed3 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 16:50:34 -0500
Subject: [PATCH 12/20] =?UTF-8?q?Black=20and=20flake=208=20fight=20with=20?=
 =?UTF-8?q?eachother=20=F0=9F=92=A5?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index b51ce18..bde59d8 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -299,4 +299,5 @@ def _get_database(uri, tls):
         return client.get_database()
 
 
-class DuplicateUniqueID(Exception): ...
+class DuplicateUniqueID(Exception): 
+    ...

From 08d454beae2f22e0ef9486a135395e291a89dfd1 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 16:52:58 -0500
Subject: [PATCH 13/20] =?UTF-8?q?Black=20vs=20Flake8,=20round=202=20?=
 =?UTF-8?q?=F0=9F=A5=8A?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index bde59d8..07d08ed 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -299,5 +299,5 @@ def _get_database(uri, tls):
         return client.get_database()
 
 
-class DuplicateUniqueID(Exception): 
-    ...
+class DuplicateUniqueID(Exception):
+    ...
\ No newline at end of file

From 2bcf7b02273a2f69f38d8dde9de8fa830c58edbf Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 16:53:59 -0500
Subject: [PATCH 14/20] =?UTF-8?q?round=203=20=F0=9F=A4=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 07d08ed..d9c9f2b 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -300,4 +300,4 @@ def _get_database(uri, tls):
 
 
 class DuplicateUniqueID(Exception):
-    ...
\ No newline at end of file
+    ...

From 82298f58be58223e8b6e0ab544fff15b42df1b17 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Fri, 2 Feb 2024 17:02:28 -0500
Subject: [PATCH 15/20] =?UTF-8?q?Prevent=20=20alteration=20of=20foreign=20?=
 =?UTF-8?q?key=20fields=20=F0=9F=91=AE=E2=80=8D=E2=99=80=EF=B8=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index d9c9f2b..563e342 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -203,6 +203,14 @@ def update(self, name, doc):
                 current_col.insert_one(doc)
             else:
                 old.pop("_id")
+                # Field Saftey Enforcement : Prevent restricted fields from changing
+                restricted_fields = ["run_start"]
+                for field in restricted_fields:
+                    if field in old and field in doc:
+                        if old[field] != doc[field]:
+                            raise ValueError(
+                                f"Field '{field}' is restricted and cannot be changed."
+                            )
                 target_uid_docs = revisions_col.find({"document.uid": doc["uid"]})
                 cur = target_uid_docs.sort([("revision", pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()

From defb806140ed662406a2de76ac480f634f9ebf34 Mon Sep 17 00:00:00 2001
From: Kari Barry <kezzsim@gmail.com>
Date: Mon, 5 Feb 2024 12:41:43 -0500
Subject: [PATCH 16/20] Better living through consistent documentation and
 python specific optimizations

Co-authored-by: Dan Allan <daniel.b.allan@gmail.com>
---
 suitcase/mongo_normalized/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index 563e342..dcd2f8a 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -182,13 +182,13 @@ def update(self, name, doc):
         Parameters
         ----------
 
-        name: {'start'} OR {'stop'} OR {'descriptor'}
+        name: {'start', 'stop', 'descriptor'}
             The type of document being updated.
         doc: dict
             The new version of the document. Its uid will be used to match it
             to the current version, the one to be updated.
         """
-        if name in ["start", "stop", "descriptor"]:
+        if name in {"start", "stop", "descriptor"}:
             event_model.schema_validators[
                 getattr(event_model.DocumentNames, name)
             ].validate(doc)

From b5101ed0b556b057f7d5d8d6072f6b07b4e1b2b3 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Mon, 5 Feb 2024 12:57:56 -0500
Subject: [PATCH 17/20] Permit removing the `start_id` but not in validator
 (yet)

---
 suitcase/mongo_normalized/__init__.py       | 5 +++--
 suitcase/mongo_normalized/tests/fixtures.py | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index dcd2f8a..e7c1da8 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -208,7 +208,9 @@ def update(self, name, doc):
                 for field in restricted_fields:
                     if field in old and field in doc:
                         if old[field] != doc[field]:
-                            raise ValueError(
+                            # Allow the removal of restricted fields, but not update
+                            if doc[field] != None:
+                                raise ValueError(
                                 f"Field '{field}' is restricted and cannot be changed."
                             )
                 target_uid_docs = revisions_col.find({"document.uid": doc["uid"]})
@@ -224,7 +226,6 @@ def update(self, name, doc):
         else:
             raise NotImplementedError(
                 f"Updating a {name} document is not currently supported. "
-                f"Only updates to 'start' documents are supported."
             )
 
     def start(self, doc):
diff --git a/suitcase/mongo_normalized/tests/fixtures.py b/suitcase/mongo_normalized/tests/fixtures.py
index 2fca399..7c77189 100644
--- a/suitcase/mongo_normalized/tests/fixtures.py
+++ b/suitcase/mongo_normalized/tests/fixtures.py
@@ -2,6 +2,7 @@
 # intake-bluesky-mongo) to import and reuse this fixtures without importing
 # *all* the fixtures used in conftest and the dependencies that they carry.
 import mongomock
+import pymongo
 import pytest
 import uuid
 
@@ -10,8 +11,8 @@
 def db_factory(request):
     def inner():
         database_name = f'test-{str(uuid.uuid4())}'
-        uri = 'mongodb://localhost:27017/'
-        client = mongomock.MongoClient(uri)
+        uri = 'mongodb://dbroker:secret@localhost:27017/test_database?authSource=admin'
+        client = pymongo.MongoClient(uri, False)
 
         def drop():
             client.drop_database(database_name)

From e3c97281d9c5948399e4527b69a4ba21409a446d Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Mon, 5 Feb 2024 13:07:58 -0500
Subject: [PATCH 18/20] =?UTF-8?q?Rollback=20fixtures=20and=20clean=20up=20?=
 =?UTF-8?q?=F0=9F=A7=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py       | 6 +++---
 suitcase/mongo_normalized/tests/fixtures.py | 7 +++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index e7c1da8..d8d8a49 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -209,10 +209,10 @@ def update(self, name, doc):
                     if field in old and field in doc:
                         if old[field] != doc[field]:
                             # Allow the removal of restricted fields, but not update
-                            if doc[field] != None:
+                            if doc[field] is not None:
                                 raise ValueError(
-                                f"Field '{field}' is restricted and cannot be changed."
-                            )
+                                    f"Field '{field}' is restricted and cannot be changed."
+                                )
                 target_uid_docs = revisions_col.find({"document.uid": doc["uid"]})
                 cur = target_uid_docs.sort([("revision", pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()
diff --git a/suitcase/mongo_normalized/tests/fixtures.py b/suitcase/mongo_normalized/tests/fixtures.py
index 7c77189..521c82f 100644
--- a/suitcase/mongo_normalized/tests/fixtures.py
+++ b/suitcase/mongo_normalized/tests/fixtures.py
@@ -2,7 +2,6 @@
 # intake-bluesky-mongo) to import and reuse this fixtures without importing
 # *all* the fixtures used in conftest and the dependencies that they carry.
 import mongomock
-import pymongo
 import pytest
 import uuid
 
@@ -11,12 +10,12 @@
 def db_factory(request):
     def inner():
         database_name = f'test-{str(uuid.uuid4())}'
-        uri = 'mongodb://dbroker:secret@localhost:27017/test_database?authSource=admin'
-        client = pymongo.MongoClient(uri, False)
+        uri = 'mongodb://localhost:27017/'
+        client = mongomock.MongoClient(uri)
 
         def drop():
             client.drop_database(database_name)
 
         request.addfinalizer(drop)
         return client[database_name]
-    return inner
+    return inner
\ No newline at end of file

From b566dfa8c8f664de21d92e1409740b2e77701ed2 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Mon, 5 Feb 2024 13:08:54 -0500
Subject: [PATCH 19/20] =?UTF-8?q?Lint=20=F0=9F=A7=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/tests/fixtures.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/suitcase/mongo_normalized/tests/fixtures.py b/suitcase/mongo_normalized/tests/fixtures.py
index 521c82f..2fca399 100644
--- a/suitcase/mongo_normalized/tests/fixtures.py
+++ b/suitcase/mongo_normalized/tests/fixtures.py
@@ -18,4 +18,4 @@ def drop():
 
         request.addfinalizer(drop)
         return client[database_name]
-    return inner
\ No newline at end of file
+    return inner

From 34f19df50553d6bd6e6b977e60b0969f7a031484 Mon Sep 17 00:00:00 2001
From: kari Barry <kezzsim@gmail.com>
Date: Mon, 5 Feb 2024 13:58:55 -0500
Subject: [PATCH 20/20] =?UTF-8?q?Actually,=20let's=20not=20edit=20`run=5Fs?=
 =?UTF-8?q?tart`;=20tis=20a=20silly=20prop=20=F0=9F=8F=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 suitcase/mongo_normalized/__init__.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/suitcase/mongo_normalized/__init__.py b/suitcase/mongo_normalized/__init__.py
index d8d8a49..af68543 100644
--- a/suitcase/mongo_normalized/__init__.py
+++ b/suitcase/mongo_normalized/__init__.py
@@ -208,11 +208,9 @@ def update(self, name, doc):
                 for field in restricted_fields:
                     if field in old and field in doc:
                         if old[field] != doc[field]:
-                            # Allow the removal of restricted fields, but not update
-                            if doc[field] is not None:
-                                raise ValueError(
-                                    f"Field '{field}' is restricted and cannot be changed."
-                                )
+                            raise ValueError(
+                                f"Field '{field}' is restricted and cannot be changed."
+                            )
                 target_uid_docs = revisions_col.find({"document.uid": doc["uid"]})
                 cur = target_uid_docs.sort([("revision", pymongo.DESCENDING)]).limit(1)
                 wrapped = dict()