From 7510a6ffaef7c9f102fa2a482cca23def0df023d Mon Sep 17 00:00:00 2001 From: Jesse Buonanno Date: Mon, 12 Aug 2019 15:44:57 -0400 Subject: [PATCH 1/6] Set zip_safe to False, as not all libaries are zip_safe compliant when installing through setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 444f871..64f3e78 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ author_email=EMAIL, python_requires=REQUIRES_PYTHON, url=URL, + zip_safe=False, classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', From 12999f8cc77461d48c10fc2e657cc81e88f6b50f Mon Sep 17 00:00:00 2001 From: Jesse Buonanno Date: Tue, 13 Aug 2019 15:14:37 -0400 Subject: [PATCH 2/6] Added venv files to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 72461e2..23fdaba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # Distribution / packaging .Python build/ +bin/ develop-eggs/ dist/ downloads/ @@ -34,6 +35,7 @@ MANIFEST # Installer logs pip-log.txt pip-delete-this-directory.txt +pip-selfcheck.json # Unit test / coverage reports htmlcov/ @@ -89,6 +91,7 @@ venv/ ENV/ env.bak/ venv.bak/ +pyvenv.cfg # Spyder project settings .spyderproject From 6ab935a06fbcefa6868b08097164e5ea76b83ac7 Mon Sep 17 00:00:00 2001 From: Jesse Buonanno Date: Wed, 14 Aug 2019 16:01:55 -0400 Subject: [PATCH 3/6] Added directory monitoring functionality Additionally, refreshing / will update the DB --- src/thumbtack/__init__.py | 18 ++++++++++++++- src/thumbtack/utils.py | 48 +++++++++++++++++++++++++++++++++++++++ src/thumbtack/views.py | 4 +++- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/thumbtack/__init__.py b/src/thumbtack/__init__.py index ffa0082..00a83f8 100644 --- a/src/thumbtack/__init__.py +++ b/src/thumbtack/__init__.py @@ -1,5 +1,7 @@ import logging import os +import time +import threading from pathlib import Path from pkg_resources import get_distribution, DistributionNotFound @@ -10,7 +12,7 @@ from flask_restful import Api from .resources import Mount, SupportedLibraries -from .utils import init_db +from .utils import init_db, monitor_image_dir from .views import main @@ -20,6 +22,18 @@ __version__ = 'Could not find version' +class ditectory_monitoring(threading.Thread): + def __init__(self, app): + threading.Thread.__init__(self) + self.app = app + + def run(self): + with self.app.app_context(): + while True: + time.sleep(3) + monitor_image_dir() + + def create_app(image_dir=None, database=None): app = Flask(__name__) app.config.from_object('thumbtack.config') @@ -114,4 +128,6 @@ def configure_logging(app): @click.option('--db', 'database', help='SQLite database to store mount state') def start_app(debug, host, port, image_dir, database): app = create_app(image_dir=image_dir, database=database) + ditectory_monitoring_thread = ditectory_monitoring(app) + ditectory_monitoring_thread.start() app.run(debug=debug, host=host, port=port) diff --git a/src/thumbtack/utils.py b/src/thumbtack/utils.py index 33ed1e6..4146e7b 100644 --- a/src/thumbtack/utils.py +++ b/src/thumbtack/utils.py @@ -4,6 +4,7 @@ import sqlite3 import subprocess import sys +import threading from pathlib import Path @@ -340,6 +341,53 @@ def insert_images(): insert_image(full_path) +def remove_image(full_path): + full_path_str = str(full_path) + disk_image = query_db("SELECT * FROM disk_images WHERE full_path = ?", [full_path_str], one=True) + + if disk_image: + current_app.logger.debug('Removing disk image from DB: {}'.format(full_path)) + sql = "DELETE from disk_images WHERE (full_path) = (?)" + update_or_insert_db(sql, [full_path_str]) + else: + current_app.logger.debug('({}) is on disk: {}'.format(disk_image['id'], full_path)) + + +def remove_images(): + images_in_db = get_images() + full_path_filenames = [] + + for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']): + for filename in files: + full_path = Path(root, filename) + full_path_filenames.append(full_path) + + # If image in DB is not on disk, remove it from DB + [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames] + + +# More efficent than calling insert_images then remove_images which will scan all files twice _and_ hit disk +def monitor_image_dir(): + full_path_filenames = [] + + for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']): + for filename in files: + + full_path = Path(root, filename) + full_path_filenames.append(full_path) + print(full_path) + + if check_ignored(full_path): + continue + + if not filename.startswith('.') and full_path.is_file(): + insert_image(full_path) + + images_in_db = get_images() + # If image in DB is not on disk, remove it from DB + [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames] + + def get_db(): db = getattr(g, '_database', None) if db is None: diff --git a/src/thumbtack/views.py b/src/thumbtack/views.py index aa0b08e..3160670 100644 --- a/src/thumbtack/views.py +++ b/src/thumbtack/views.py @@ -2,7 +2,7 @@ from flask import Blueprint, current_app, redirect, render_template, request -from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count +from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count, monitor_image_dir from .exceptions import UnexpectedDiskError, NoMountableVolumesError main = Blueprint('', __name__) @@ -23,6 +23,8 @@ def index(): else: unsupported_mount_types.append(mount_type) + # On refresh, update DB to match what's on disk + monitor_image_dir() images = get_images() return render_template('index.html', From 79de63557728d071c157590f9cab108109a26df2 Mon Sep 17 00:00:00 2001 From: Jesse Buonanno Date: Wed, 14 Aug 2019 16:10:57 -0400 Subject: [PATCH 4/6] Fixed directory_monitoring typo --- src/thumbtack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/thumbtack/__init__.py b/src/thumbtack/__init__.py index 00a83f8..b723153 100644 --- a/src/thumbtack/__init__.py +++ b/src/thumbtack/__init__.py @@ -22,7 +22,7 @@ __version__ = 'Could not find version' -class ditectory_monitoring(threading.Thread): +class directory_monitoring(threading.Thread): def __init__(self, app): threading.Thread.__init__(self) self.app = app From 6f26b84ecede5974aa7c90c1f91e073ecde67d14 Mon Sep 17 00:00:00 2001 From: Jesse Buonanno Date: Wed, 14 Aug 2019 16:15:00 -0400 Subject: [PATCH 5/6] Removed dev printlines Fixed more typos --- src/thumbtack/__init__.py | 4 ++-- src/thumbtack/utils.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/thumbtack/__init__.py b/src/thumbtack/__init__.py index b723153..2badec1 100644 --- a/src/thumbtack/__init__.py +++ b/src/thumbtack/__init__.py @@ -128,6 +128,6 @@ def configure_logging(app): @click.option('--db', 'database', help='SQLite database to store mount state') def start_app(debug, host, port, image_dir, database): app = create_app(image_dir=image_dir, database=database) - ditectory_monitoring_thread = ditectory_monitoring(app) - ditectory_monitoring_thread.start() + directory_monitoring_thread = directory_monitoring(app) + directory_monitoring_thread.start() app.run(debug=debug, host=host, port=port) diff --git a/src/thumbtack/utils.py b/src/thumbtack/utils.py index 4146e7b..a1d4e98 100644 --- a/src/thumbtack/utils.py +++ b/src/thumbtack/utils.py @@ -4,7 +4,6 @@ import sqlite3 import subprocess import sys -import threading from pathlib import Path @@ -375,7 +374,6 @@ def monitor_image_dir(): full_path = Path(root, filename) full_path_filenames.append(full_path) - print(full_path) if check_ignored(full_path): continue From 8d74c916cfe2ed30995087e818b9e19539d7a9c9 Mon Sep 17 00:00:00 2001 From: Jared Ondricek Date: Fri, 23 Aug 2019 13:56:37 -0400 Subject: [PATCH 6/6] Enable directory mounting for nginx and gunicorn --- Vagrantfile | 1 + provisioning/install.sh | 3 +++ src/thumbtack/__init__.py | 30 ++++++++------------------- src/thumbtack/directory_monitoring.py | 16 ++++++++++++++ src/thumbtack/views.py | 2 ++ 5 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 src/thumbtack/directory_monitoring.py diff --git a/Vagrantfile b/Vagrantfile index 0a9a859..3f4d29b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,6 +4,7 @@ Vagrant.configure("2") do |config| config.vm.box = "bento/ubuntu-16.04" + config.vm.network "forwarded_port", guest: 80, host: 8080, id: "nginx" config.vm.network "forwarded_port", guest: 5000, host: 5000, id: "Thumbtack-dev" config.vm.network "forwarded_port", guest: 8208, host: 8208, id: "Thumbtack" config.vm.provider "virtualbox" do |v| diff --git a/provisioning/install.sh b/provisioning/install.sh index d7b4dcb..7f091f6 100644 --- a/provisioning/install.sh +++ b/provisioning/install.sh @@ -35,6 +35,9 @@ sudo apt install -y python3-pip \ unzip \ zip +# install tools for testing a production setup +sudo apt install -y nginx + cd /vagrant # install the thumbtack python library in development mode sudo pip3 install -e .[dev] diff --git a/src/thumbtack/__init__.py b/src/thumbtack/__init__.py index 2badec1..a25c5d1 100644 --- a/src/thumbtack/__init__.py +++ b/src/thumbtack/__init__.py @@ -1,7 +1,5 @@ import logging import os -import time -import threading from pathlib import Path from pkg_resources import get_distribution, DistributionNotFound @@ -11,6 +9,7 @@ from flask import Flask, current_app from flask_restful import Api +from .directory_monitoring import DirectoryMonitoring from .resources import Mount, SupportedLibraries from .utils import init_db, monitor_image_dir from .views import main @@ -22,18 +21,6 @@ __version__ = 'Could not find version' -class directory_monitoring(threading.Thread): - def __init__(self, app): - threading.Thread.__init__(self) - self.app = app - - def run(self): - with self.app.app_context(): - while True: - time.sleep(3) - monitor_image_dir() - - def create_app(image_dir=None, database=None): app = Flask(__name__) app.config.from_object('thumbtack.config') @@ -104,16 +91,17 @@ def before_first_request(): def configure_logging(app): - if not app.debug: - # In production mode, add log handler to sys.stderr. - app.logger.setLevel(logging.DEBUG) + formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s") - formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s") + if app.debug: + app.logger.setLevel(logging.DEBUG) + else: + app.logger.setLevel(logging.INFO) + # In production mode, add log handler to sys.stderr. shandler = logging.StreamHandler() - shandler.setLevel(logging.DEBUG) + shandler.setLevel(logging.INFO) shandler.setFormatter(formatter) - # app.logger.addHandler(shandler) @@ -128,6 +116,6 @@ def configure_logging(app): @click.option('--db', 'database', help='SQLite database to store mount state') def start_app(debug, host, port, image_dir, database): app = create_app(image_dir=image_dir, database=database) - directory_monitoring_thread = directory_monitoring(app) + directory_monitoring_thread = DirectoryMonitoring(app) directory_monitoring_thread.start() app.run(debug=debug, host=host, port=port) diff --git a/src/thumbtack/directory_monitoring.py b/src/thumbtack/directory_monitoring.py new file mode 100644 index 0000000..d34db18 --- /dev/null +++ b/src/thumbtack/directory_monitoring.py @@ -0,0 +1,16 @@ +import threading +import time + +from .utils import monitor_image_dir + + +class DirectoryMonitoring(threading.Thread): + def __init__(self, app): + threading.Thread.__init__(self) + self.app = app + + def run(self): + with self.app.app_context(): + while True: + time.sleep(3) + monitor_image_dir() diff --git a/src/thumbtack/views.py b/src/thumbtack/views.py index 3160670..e1f1212 100644 --- a/src/thumbtack/views.py +++ b/src/thumbtack/views.py @@ -23,9 +23,11 @@ def index(): else: unsupported_mount_types.append(mount_type) + current_app.logger.debug('-------------- Getting images!!! --------------') # On refresh, update DB to match what's on disk monitor_image_dir() images = get_images() + current_app.logger.debug('-------------- Got images!!! --------------') return render_template('index.html', supported_mount_types=supported_mount_types,