diff --git a/.gitignore b/.gitignore index 72461e2..23fdaba 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ # Distribution / packaging .Python build/ +bin/ develop-eggs/ dist/ downloads/ @@ -34,6 +35,7 @@ MANIFEST # Installer logs pip-log.txt pip-delete-this-directory.txt +pip-selfcheck.json # Unit test / coverage reports htmlcov/ @@ -89,6 +91,7 @@ venv/ ENV/ env.bak/ venv.bak/ +pyvenv.cfg # Spyder project settings .spyderproject diff --git a/Vagrantfile b/Vagrantfile index 0a9a859..3f4d29b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,6 +4,7 @@ Vagrant.configure("2") do |config| config.vm.box = "bento/ubuntu-16.04" + config.vm.network "forwarded_port", guest: 80, host: 8080, id: "nginx" config.vm.network "forwarded_port", guest: 5000, host: 5000, id: "Thumbtack-dev" config.vm.network "forwarded_port", guest: 8208, host: 8208, id: "Thumbtack" config.vm.provider "virtualbox" do |v| diff --git a/provisioning/install.sh b/provisioning/install.sh index d7b4dcb..7f091f6 100644 --- a/provisioning/install.sh +++ b/provisioning/install.sh @@ -35,6 +35,9 @@ sudo apt install -y python3-pip \ unzip \ zip +# install tools for testing a production setup +sudo apt install -y nginx + cd /vagrant # install the thumbtack python library in development mode sudo pip3 install -e .[dev] diff --git a/setup.py b/setup.py index 444f871..64f3e78 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ author_email=EMAIL, python_requires=REQUIRES_PYTHON, url=URL, + zip_safe=False, classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/src/thumbtack/__init__.py b/src/thumbtack/__init__.py index ffa0082..a25c5d1 100644 --- a/src/thumbtack/__init__.py +++ b/src/thumbtack/__init__.py @@ -9,8 +9,9 @@ from flask import Flask, current_app from flask_restful import Api +from .directory_monitoring import DirectoryMonitoring from .resources import Mount, SupportedLibraries -from .utils import init_db +from .utils import init_db, monitor_image_dir from .views import main @@ -90,16 +91,17 @@ def before_first_request(): def configure_logging(app): - if not app.debug: - # In production mode, add log handler to sys.stderr. - app.logger.setLevel(logging.DEBUG) + formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s") - formatter = logging.Formatter("[%(asctime)s] %(levelname)s in %(name)s.%(module)s: %(message)s") + if app.debug: + app.logger.setLevel(logging.DEBUG) + else: + app.logger.setLevel(logging.INFO) + # In production mode, add log handler to sys.stderr. shandler = logging.StreamHandler() - shandler.setLevel(logging.DEBUG) + shandler.setLevel(logging.INFO) shandler.setFormatter(formatter) - # app.logger.addHandler(shandler) @@ -114,4 +116,6 @@ def configure_logging(app): @click.option('--db', 'database', help='SQLite database to store mount state') def start_app(debug, host, port, image_dir, database): app = create_app(image_dir=image_dir, database=database) + directory_monitoring_thread = DirectoryMonitoring(app) + directory_monitoring_thread.start() app.run(debug=debug, host=host, port=port) diff --git a/src/thumbtack/directory_monitoring.py b/src/thumbtack/directory_monitoring.py new file mode 100644 index 0000000..d34db18 --- /dev/null +++ b/src/thumbtack/directory_monitoring.py @@ -0,0 +1,16 @@ +import threading +import time + +from .utils import monitor_image_dir + + +class DirectoryMonitoring(threading.Thread): + def __init__(self, app): + threading.Thread.__init__(self) + self.app = app + + def run(self): + with self.app.app_context(): + while True: + time.sleep(3) + monitor_image_dir() diff --git a/src/thumbtack/utils.py b/src/thumbtack/utils.py index 33ed1e6..a1d4e98 100644 --- a/src/thumbtack/utils.py +++ b/src/thumbtack/utils.py @@ -340,6 +340,52 @@ def insert_images(): insert_image(full_path) +def remove_image(full_path): + full_path_str = str(full_path) + disk_image = query_db("SELECT * FROM disk_images WHERE full_path = ?", [full_path_str], one=True) + + if disk_image: + current_app.logger.debug('Removing disk image from DB: {}'.format(full_path)) + sql = "DELETE from disk_images WHERE (full_path) = (?)" + update_or_insert_db(sql, [full_path_str]) + else: + current_app.logger.debug('({}) is on disk: {}'.format(disk_image['id'], full_path)) + + +def remove_images(): + images_in_db = get_images() + full_path_filenames = [] + + for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']): + for filename in files: + full_path = Path(root, filename) + full_path_filenames.append(full_path) + + # If image in DB is not on disk, remove it from DB + [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames] + + +# More efficent than calling insert_images then remove_images which will scan all files twice _and_ hit disk +def monitor_image_dir(): + full_path_filenames = [] + + for root, dirs, files in os.walk(current_app.config['IMAGE_DIR']): + for filename in files: + + full_path = Path(root, filename) + full_path_filenames.append(full_path) + + if check_ignored(full_path): + continue + + if not filename.startswith('.') and full_path.is_file(): + insert_image(full_path) + + images_in_db = get_images() + # If image in DB is not on disk, remove it from DB + [remove_image(image["full_path"]) for image in images_in_db if Path(image["full_path"]) not in full_path_filenames] + + def get_db(): db = getattr(g, '_database', None) if db is None: diff --git a/src/thumbtack/views.py b/src/thumbtack/views.py index aa0b08e..e1f1212 100644 --- a/src/thumbtack/views.py +++ b/src/thumbtack/views.py @@ -2,7 +2,7 @@ from flask import Blueprint, current_app, redirect, render_template, request -from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count +from .utils import get_supported_libraries, get_images, mount_image, unmount_image, get_ref_count, monitor_image_dir from .exceptions import UnexpectedDiskError, NoMountableVolumesError main = Blueprint('', __name__) @@ -23,7 +23,11 @@ def index(): else: unsupported_mount_types.append(mount_type) + current_app.logger.debug('-------------- Getting images!!! --------------') + # On refresh, update DB to match what's on disk + monitor_image_dir() images = get_images() + current_app.logger.debug('-------------- Got images!!! --------------') return render_template('index.html', supported_mount_types=supported_mount_types,