utils/json: Generalize JSON extra types support (#163)

qtoggleserver/drivers/persist/json.py

@@ -253,7 +253,7 @@ def _load(self) -> UnindexedData:
         try:
             with open(self._file_path, 'rb') as f:
                 data = f.read()
-                return json_utils.loads(data, allow_extended_types=True)
+                return json_utils.loads(data, extra_types=json_utils.EXTRA_TYPES_EXTENDED)
         except Exception as e:
             if not self._use_backup:
                 raise
@@ -266,7 +266,7 @@ def _load(self) -> UnindexedData:
                 logger.warning('loading from backup %s', backup_file_path)
 
                 with open(backup_file_path, 'rb') as f:
-                    return json_utils.loads(f.read(), allow_extended_types=True)
+                    return json_utils.loads(f.read(), extra_types=json_utils.EXTRA_TYPES_EXTENDED)
 
         return {}
 
@@ -284,7 +284,9 @@ def _save(self, data: UnindexedData) -> None:
         logger.debug('saving to %s', self._file_path)
 
         with open(self._file_path, 'wb') as f:
-            data = json_utils.dumps(data, allow_extended_types=True, indent=4 if self._pretty_format else None)
+            data = json_utils.dumps(
+                data, extra_types=json_utils.EXTRA_TYPES_EXTENDED, indent=4 if self._pretty_format else None
+            )
             f.write(data.encode())
 
     @staticmethod

qtoggleserver/drivers/persist/redis.py

@@ -287,11 +287,11 @@ def _record_to_db(cls, record: Record) -> GenericJSONDict:
 
     @staticmethod
     def _value_to_db(value: Any) -> str:
-        return json_utils.dumps(value, allow_extended_types=True)
+        return json_utils.dumps(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED)
 
     @staticmethod
     def _value_from_db(value: str) -> Any:
-        return json_utils.loads(value, allow_extended_types=True)
+        return json_utils.loads(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED)
 
     @staticmethod
     def _make_record_key(collection: str, id_: Id) -> str:

qtoggleserver/frontend/js/devices/add-device-form.js

@@ -90,10 +90,7 @@ class AddDeviceForm extends PageForm {
         }).catch(function (error) {
 
             /* Retry with /api path, which should be a default location for qToggleServer implementations */
-            if (error instanceof BaseAPI.APIError &&
-                RETRY_API_ERROR_CODES.includes(error.code) &&
-                url.path === '/') {
-
+            if (error instanceof BaseAPI.APIError && error.status === 404 && url.path === '/') {
                 logger.debug('retrying with /api suffix')
                 url.path = '/api'
                 data.url = url.toString()

qtoggleserver/persist/__init__.py

@@ -59,7 +59,7 @@ async def query(
             'querying %s (%s) where %s (sort=%s, limit=%s)',
             collection,
             json_utils.dumps(fields) if fields else 'all fields',
-            json_utils.dumps(filt, allow_extended_types=True),
+            json_utils.dumps(filt, extra_types=json_utils.EXTRA_TYPES_EXTENDED),
             json_utils.dumps(sort),
             json_utils.dumps(limit),
         )
@@ -123,7 +123,7 @@ async def set_value(name: str, value: Any) -> None:
     considering the fist (and only) record."""
 
     if logger.getEffectiveLevel() <= logging.DEBUG:
-        logger.debug('setting %s to %s', name, json_utils.dumps(value, allow_extended_types=True))
+        logger.debug('setting %s to %s', name, json_utils.dumps(value, extra_types=json_utils.EXTRA_TYPES_EXTENDED))
 
     driver = await _get_driver()
     record = {'value': value}
@@ -152,7 +152,9 @@ async def insert(collection: str, record: Record) -> Id:
     Return the associated record ID."""
 
     if logger.getEffectiveLevel() <= logging.DEBUG:
-        logger.debug('inserting %s into %s', json_utils.dumps(record, allow_extended_types=True), collection)
+        logger.debug(
+            'inserting %s into %s', json_utils.dumps(record, extra_types=json_utils.EXTRA_TYPES_EXTENDED), collection
+        )
 
     driver = await _get_driver()
     return await driver.insert(collection, record)
@@ -170,8 +172,8 @@ async def update(collection: str, record_part: Record, filt: Optional[dict[str,
         logger.debug(
             'updating %s where %s with %s',
             collection,
-            json_utils.dumps(filt or {}, allow_extended_types=True),
-            json_utils.dumps(record_part, allow_extended_types=True)
+            json_utils.dumps(filt or {}, extra_types=json_utils.EXTRA_TYPES_EXTENDED),
+            json_utils.dumps(record_part, extra_types=json_utils.EXTRA_TYPES_EXTENDED)
         )
 
     driver = await _get_driver()
@@ -191,7 +193,7 @@ async def replace(collection: str, id_: Id, record: Record) -> bool:
         logger.debug(
             'replacing record with id %s with %s in %s',
             id_,
-            json_utils.dumps(record, allow_extended_types=True),
+            json_utils.dumps(record, extra_types=json_utils.EXTRA_TYPES_EXTENDED),
             collection
         )
 
@@ -219,7 +221,10 @@ async def remove(collection: str, filt: Optional[dict[str, Any]] = None) -> int:
     Return the total number of records that were removed."""
 
     if logger.getEffectiveLevel() <= logging.DEBUG:
-        logger.debug('removing from %s where %s', collection, json_utils.dumps(filt or {}, allow_extended_types=True))
+        logger.debug(
+            'removing from %s where %s',
+            collection, json_utils.dumps(filt or {}, extra_types=json_utils.EXTRA_TYPES_EXTENDED),
+        )
 
     driver = await _get_driver()
     count = await driver.remove(collection, filt or {})

qtoggleserver/utils/json.py

@@ -16,6 +16,15 @@
 DATE_TYPE = '__d'
 DATETIME_TYPE = '__dt'
 
+DATETIME_FORMAT_ISO = '%Y-%m-%dT%H:%M:%SZ'
+DATE_FORMAT_ISO = '%Y-%m-%d'
+DATETIME_FORMAT_ISO_LEN = len(DATETIME_FORMAT_ISO)
+DATE_FORMAT_ISO_LEN = len(DATE_FORMAT_ISO)
+
+EXTRA_TYPES_NONE = ''
+EXTRA_TYPES_ISO = 'iso'
+EXTRA_TYPES_EXTENDED = 'extended'
+
 
 def _replace_nan_inf_rec(obj: Any, replace_value: Any) -> Any:
     if isinstance(obj, dict):
@@ -54,7 +63,18 @@ def _resolve_refs_rec(obj: Any, root_obj: Any) -> Any:
     return obj
 
 
-def encode_default_json(obj: Any) -> Any:
+def encode_default_json_iso(obj: Any) -> Any:
+    if isinstance(obj, datetime.datetime):
+        return obj.strftime(DATETIME_FORMAT_ISO)
+    elif isinstance(obj, datetime.date):
+        return obj.strftime(DATE_FORMAT)
+    elif isinstance(obj, (set, tuple)):
+        return list(obj)
+    else:
+        raise TypeError()
+
+
+def encode_default_json_extended(obj: Any) -> Any:
     if isinstance(obj, datetime.datetime):
         return {
             TYPE_FIELD: DATETIME_TYPE,
@@ -71,7 +91,24 @@ def encode_default_json(obj: Any) -> Any:
         raise TypeError()
 
 
-def decode_json_hook(obj: dict) -> Any:
+def decode_json_hook_iso(obj: dict) -> Any:
+    for k, v in obj.items():
+        if isinstance(v, str):
+            if len(v) == DATETIME_FORMAT_ISO_LEN:
+                try:
+                    obj[k] = datetime.datetime.strptime(v, DATETIME_FORMAT_ISO)
+                except ValueError:
+                    pass
+            elif len(v) == DATE_FORMAT_ISO_LEN:
+                try:
+                    obj[k] = datetime.datetime.strptime(v, DATE_FORMAT_ISO).date()
+                except ValueError:
+                    pass
+
+    return obj
+
+
+def decode_json_hook_extended(obj: dict) -> Any:
     __t = obj.get(TYPE_FIELD)
     if __t is not None:
         __v = obj.get(VALUE_FIELD)
@@ -89,7 +126,7 @@ def decode_json_hook(obj: dict) -> Any:
     return obj
 
 
-def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str:
+def dumps(obj: Any, extra_types: str = EXTRA_TYPES_NONE, **kwargs) -> str:
     # Treat primitive types separately to gain just a bit of performance
     if isinstance(obj, str):
         return '"' + obj + '"'
@@ -103,8 +140,10 @@ def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str:
     elif obj is None:
         return 'null'
     else:
-        if allow_extended_types:
-            return json.dumps(obj, default=encode_default_json, allow_nan=True, **kwargs)
+        if extra_types == EXTRA_TYPES_EXTENDED:
+            return json.dumps(obj, default=encode_default_json_extended, allow_nan=True, **kwargs)
+        elif extra_types == EXTRA_TYPES_ISO:
+            return json.dumps(obj, default=encode_default_json_iso, allow_nan=False, **kwargs)
         else:
             try:
                 return json.dumps(obj, allow_nan=False, **kwargs)
@@ -114,10 +153,15 @@ def dumps(obj: Any, allow_extended_types: bool = False, **kwargs) -> str:
                 return json.dumps(obj, allow_nan=False, **kwargs)
 
 
-def loads(s: Union[str, bytes], resolve_refs: bool = False, allow_extended_types: bool = False, **kwargs) -> Any:
-    object_hook = decode_json_hook if allow_extended_types else None
-    obj = json.loads(s, object_hook=object_hook, **kwargs)
+def loads(s: Union[str, bytes], resolve_refs: bool = False, extra_types: str = EXTRA_TYPES_NONE, **kwargs) -> Any:
+    if extra_types == EXTRA_TYPES_EXTENDED:
+        object_hook = decode_json_hook_extended
+    elif extra_types == EXTRA_TYPES_ISO:
+        object_hook = decode_json_hook_iso
+    else:
+        object_hook = None
 
+    obj = json.loads(s, object_hook=object_hook, **kwargs)
     if resolve_refs:
         obj = _resolve_refs_rec(obj, root_obj=obj)