Merge pull request #7 from mitre/directory_monitoring

Add Directory Monitoring

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 build/
+bin/
 develop-eggs/
 dist/
 downloads/
@@ -34,6 +35,7 @@ MANIFEST
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
+pip-selfcheck.json
 
 # Unit test / coverage reports
 htmlcov/
@@ -89,6 +91,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+pyvenv.cfg
 
 # Spyder project settings
 .spyderproject

Vagrantfile

@@ -4,6 +4,7 @@
 Vagrant.configure("2") do |config|
   config.vm.box = "bento/ubuntu-16.04"
 
+  config.vm.network "forwarded_port", guest: 80, host: 8080, id: "nginx"
   config.vm.network "forwarded_port", guest: 5000, host: 5000, id: "Thumbtack-dev"
   config.vm.network "forwarded_port", guest: 8208, host: 8208, id: "Thumbtack"
   config.vm.provider "virtualbox" do |v|

provisioning/install.sh

@@ -35,6 +35,9 @@ sudo apt install -y python3-pip \
                     unzip \
                     zip
 
+# install tools for testing a production setup
+sudo apt install -y nginx
+
 cd /vagrant
 # install the thumbtack python library in development mode
 sudo pip3 install -e .[dev]

setup.py

@@ -65,6 +65,7 @@
     author_email=EMAIL,
     python_requires=REQUIRES_PYTHON,
     url=URL,
+    zip_safe=False,
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',

src/thumbtack/__init__.py

@@ -9,8 +9,9 @@
 from flask import Flask, current_app
 from flask_restful import Api
 
+from .directory_monitoring import DirectoryMonitoring
 from .resources import Mount, SupportedLibraries
-from .utils import init_db
+from .utils import init_db, monitor_image_dir
 from .views import main
 
 
@@ -90,16 +91,17 @@ def before_first_request():
 
 
 def configure_logging(app):
-    if not app.debug:
-        # In production mode, add log handler to sys.stderr.
-        app.logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s")
 
-        formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s")
+    if app.debug:
+        app.logger.setLevel(logging.DEBUG)
+    else:
+        app.logger.setLevel(logging.INFO)
 
+        # In production mode, add log handler to sys.stderr.
         shandler = logging.StreamHandler()
-        shandler.setLevel(logging.DEBUG)
+        shandler.setLevel(logging.INFO)
         shandler.setFormatter(formatter)
-
         # app.logger.addHandler(shandler)
 
 
@@ -114,4 +116,6 @@ def configure_logging(app):
 @click.option('--db', 'database', help='SQLite database to store mount state')
 def start_app(debug, host, port, image_dir, database):
     app = create_app(image_dir=image_dir, database=database)
+    directory_monitoring_thread = DirectoryMonitoring(app)
+    directory_monitoring_thread.start()
     app.run(debug=debug, host=host, port=port)

src/thumbtack/directory_monitoring.py

@@ -0,0 +1,16 @@
+import threading
+import time
+
+from .utils import monitor_image_dir
+
+
+class DirectoryMonitoring(threading.Thread):
+    def __init__(self, app):
+        threading.Thread.__init__(self)
+        self.app = app
+
+    def run(self):
+        with self.app.app_context():
+            while True:
+                time.sleep(3)
+                monitor_image_dir()

src/thumbtack/utils.py

@@ -340,6 +340,52 @@ def insert_images():
                 insert_image(full_path)
 
 
+def remove_image(full_path):
+    full_path_str = str(full_path)
+    disk_image = query_db("SELECT * FROM disk_images WHERE full_path = ?", [full_path_str], one=True)
+
+    if disk_image:
+        current_app.logger.debug('Removing disk image from DB: {}'.format(full_path))
+        sql = "DELETE from disk_images WHERE (full_path) = (?)"
+        update_or_insert_db(sql, [full_path_str])
+    else:
+        current_app.logger.debug('({}) is on disk: {}'.format(disk_image['id'], full_path))
+
+
+def remove_images():
+    images_in_db = get_images()
+    full_path_filenames = []
+
+    for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']):
+        for filename in files:
+            full_path = Path(root, filename)
+            full_path_filenames.append(full_path)
+
+    # If image in DB is not on disk, remove it from DB
+    [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames]
+
+
+# More efficent than calling insert_images then remove_images which will scan all files twice _and_ hit disk
+def monitor_image_dir():
+    full_path_filenames = []
+
+    for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']):
+        for filename in files:
+
+            full_path = Path(root, filename)
+            full_path_filenames.append(full_path)
+
+            if check_ignored(full_path):
+                continue
+
+            if not filename.startswith('.') and full_path.is_file():
+                insert_image(full_path)
+
+    images_in_db = get_images()
+    # If image in DB is not on disk, remove it from DB
+    [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames]
+
+
 def get_db():
     db = getattr(g, '_database', None)
     if db is None:

src/thumbtack/views.py

@@ -2,7 +2,7 @@
 
 from flask import Blueprint, current_app, redirect, render_template, request
 
-from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count
+from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count, monitor_image_dir
 from .exceptions import UnexpectedDiskError, NoMountableVolumesError
 
 main = Blueprint('', __name__)
@@ -23,7 +23,11 @@ def index():
         else:
             unsupported_mount_types.append(mount_type)
 
+    current_app.logger.debug('-------------- Getting images!!! --------------')
+    # On refresh, update DB to match what's on disk
+    monitor_image_dir()
     images = get_images()
+    current_app.logger.debug('-------------- Got images!!! --------------')
 
     return render_template('index.html',
                            supported_mount_types=supported_mount_types,