Abstract data from the model, save each watch data to its own directory

changedetectionio/api/api_v1.py

@@ -12,10 +12,10 @@
 # See docs/README.md for rebuilding the docs/apidoc information
 
 from . import api_schema
-from ..model import watch_base
+from ..model import WatchBase
 
 # Build a JSON Schema atleast partially based on our Watch model
-watch_base_config = watch_base()
+watch_base_config = WatchBase()
 schema = api_schema.build_watch_json_schema(watch_base_config)
 
 schema_create_watch = copy.deepcopy(schema)
@@ -52,8 +52,8 @@ def get(self, uuid):
         @apiSuccess (200) {String} OK When paused/muted/recheck operation OR full JSON object of the watch
         @apiSuccess (200) {JSON} WatchJSON JSON Full JSON object of the watch
         """
-        from copy import deepcopy
-        watch = deepcopy(self.datastore.data['watching'].get(uuid))
+        watch = self.datastore.data['watching'].get(uuid)
+
         if not watch:
             abort(404, message='No watch exists with the UUID of {}'.format(uuid))
 
@@ -75,10 +75,11 @@ def get(self, uuid):
 
         # Return without history, get that via another API call
         # Properties are not returned as a JSON, so add the required props manually
-        watch['history_n'] = watch.history_n
-        watch['last_changed'] = watch.last_changed
-        watch['viewed'] = watch.viewed
-        return watch
+        result = watch.as_dict()
+        result['history_n'] = watch.history_n
+        result['last_changed'] = watch.last_changed
+        result['viewed'] = watch.viewed
+        return result
 
     @auth.check_token
     def delete(self, uuid):

changedetectionio/blueprint/check_proxies/__init__.py

@@ -41,6 +41,7 @@ def long_task(uuid, preferred_proxy):
         now = time.time()
         try:
             processor_module = importlib.import_module("changedetectionio.processors.text_json_diff.processor")
+            # @todo can now just pass the watch here?
             update_handler = processor_module.perform_site_check(datastore=datastore,
                                                                  watch_uuid=uuid
                                                                  )

changedetectionio/flask_app.py

@@ -43,6 +43,7 @@
 from changedetectionio import html_tools, __version__
 from changedetectionio import queuedWatchMetaData
 from changedetectionio.api import api_v1
+from .store import CustomEncoder
 from .time_handler import is_within_schedule
 
 datastore = None
@@ -800,7 +801,7 @@ def edit_page(uuid):
 
             # Recast it if need be to right data Watch handler
             watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
-            datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
+            datastore.data['watching'][uuid] = watch_class(__datastore=datastore_o, default=datastore.data['watching'][uuid])
             flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
 
             # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
@@ -1613,7 +1614,7 @@ def form_share_put_watch():
         watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
         watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
 
-        watch_json = json.dumps(watch)
+        watch_json = json.dumps(watch, cls=CustomEncoder)
 
         try:
             r = requests.request(method="POST",

changedetectionio/model/App.py

@@ -10,7 +10,7 @@
 DEFAULT_SETTINGS_HEADERS_USERAGENT='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
 
 class model(dict):
-    base_config = {
+    __base_config = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
             'settings': {
@@ -60,7 +60,7 @@ class model(dict):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)
-        self.update(self.base_config)
+        self.update(self.__base_config)
 
 
 def parse_headers_from_text_file(filepath):

changedetectionio/model/Tag.py

@@ -1,8 +1,8 @@
 
-from changedetectionio.model import watch_base
+from changedetectionio.model import WatchBase
 
 
-class model(watch_base):
+class model(WatchBase):
 
     def __init__(self, *arg, **kw):
         super(model, self).__init__(*arg, **kw)

changedetectionio/model/Watch.py

@@ -1,17 +1,17 @@
 from changedetectionio.strtobool import strtobool
 from changedetectionio.safe_jinja import render as jinja_render
-from . import watch_base
 import os
 import re
 from pathlib import Path
 from loguru import logger
 
+from . import WatchBase
 from ..html_tools import TRANSLATE_WHITESPACE_TABLE
 
 # Allowable protocols, protects against javascript: etc
 # file:// is further checked by ALLOW_FILE_URI
 SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):'
-
+WATCH_DB_JSON_FILENAME = 'watch.json'
 minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3))
 mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7}
 
@@ -32,15 +32,20 @@ def is_safe_url(test_url):
     return True
 
 
-class model(watch_base):
-    __newest_history_key = None
+class model(WatchBase):
+    __datastore = None
+    __datastore_checksum = None
+
     __history_n = 0
+    __newest_history_key = None
     jitter_seconds = 0
-
+    
     def __init__(self, *arg, **kw):
-        self.__datastore_path = kw.get('datastore_path')
-        if kw.get('datastore_path'):
-            del kw['datastore_path']
+        if not kw.get('__datastore'):
+            logger.critical('No __datastore reference was set!')
+
+        self.__datastore = kw.get('__datastore')
+
         super(model, self).__init__(*arg, **kw)
         if kw.get('default'):
             self.update(kw['default'])
@@ -179,7 +184,7 @@ def history(self):
         tmp_history = {}
 
         # In the case we are only using the watch for processing without history
-        if not self.watch_data_dir:
+        if not self.__datastore or not self.watch_data_dir:
             return []
 
         # Read the history file as a dict
@@ -419,7 +424,7 @@ def snapshot_error_screenshot_ctime(self):
     @property
     def watch_data_dir(self):
         # The base dir of the watch data
-        return os.path.join(self.__datastore_path, self['uuid']) if self.__datastore_path else None
+        return os.path.join(self.__datastore.datastore_path, self['uuid']) if self.__datastore.datastore_path else None
 
     def get_error_text(self):
         """Return the text saved from a previous request that resulted in a non-200 error"""
@@ -524,6 +529,22 @@ def has_special_diff_filter_options_set(self):
         # None is set
         return False
 
+    def save_data(self):
+        import json
+        # @todo dict change?
+        # Save it to a temp file first so that if the disk is full or other error it wont corrupt (hopefully).
+
+        dest = os.path.join(self.watch_data_dir, WATCH_DB_JSON_FILENAME)
+        logger.debug(f"Saving watch {dest}")
+        try:
+            with open(dest + '.tmp', 'w') as json_file:
+                json.dump(self.as_dict(), json_file, indent=2)
+            os.replace(dest + '.tmp', dest)
+
+        except Exception as e:
+            logger.critical(f"Exception saving watch JSON {dest} - {e}")
+
+
     def save_error_text(self, contents):
         self.ensure_data_dir_exists()
         target_path = os.path.join(self.watch_data_dir, "last-error.txt")

changedetectionio/model/__init__.py

@@ -1,13 +1,14 @@
 import os
 import uuid
+from collections.abc import MutableMapping
 
 from changedetectionio import strtobool
 from changedetectionio.notification import default_notification_format_for_watch
 
-class watch_base(dict):
-
-    def __init__(self, *arg, **kw):
-        self.update({
+class WatchBase(MutableMapping):
+    __data_checksum = None
+    def __init__(self, *args, **kwargs):
+        self.__internal_dict = {
             # Custom notification content
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
@@ -127,9 +128,37 @@ def __init__(self, *arg, **kw):
             'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None,  # Run before change-detection
-        })
+        }
 
-        super(watch_base, self).__init__(*arg, **kw)
+        # Update with any provided arguments
+        self.update(*args, **kwargs)
 
         if self.get('default'):
-            del self['default']
+            del self['default']
+
+
+    # Implement abstract methods required by MutableMapping
+    def __getitem__(self, key):
+        return self.__internal_dict[key]
+
+    def __setitem__(self, key, value):
+        if key == '__datastore':
+            self.__datastore = value
+        else:
+            self.__internal_dict[key] = value
+
+    def __delitem__(self, key):
+        del self.__internal_dict[key]
+
+    def __iter__(self):
+        return iter(self.__internal_dict)
+
+    def __len__(self):
+        return len(self.__internal_dict)
+
+    # Optional: Implement additional methods for convenience
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.__internal_dict})"
+
+    def as_dict(self):
+        return self.__internal_dict