Merge pull request #24 from 4Catalyzer/boto3

Switch to boto3 for S3

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+source = flask_annex
+
+[report]
+exclude_lines =
+    pragma: no cover
+    raise NotImplementedError

.travis.yml

@@ -3,26 +3,32 @@ sudo: false
 language: python
 python:
   - "2.7"
-  - "3.4"
   - "3.5"
   - "pypy"
 
 env:
-  - TOXENV=py-base
   - TOXENV=py-s3
 
+matrix:
+  include:
+    - python: "2.7"
+      env: TOXENV=py-base
+
 cache:
   directories:
     - $HOME/.cache/pip
 
 before_install:
   - pip install -U pip
 install:
-  - pip install -U tox
+  - pip install -U codecov tox
 
 script:
   - python setup.py test
 
+after_success:
+  - codecov
+
 branches:
   only:
     - master

README.md

@@ -1,6 +1,14 @@
-# Flask-Annex [![PyPI][pypi-badge]][pypi]
+# Flask-Annex [![Travis][build-badge]][build] [![PyPI][pypi-badge]][pypi]
 Efficient integration of external storage services for
 [Flask](http://flask.pocoo.org/).
 
+[![Codecov][codecov-badge]][codecov]
+
+[build-badge]: https://img.shields.io/travis/4Catalyzer/flask-annex/master.svg
+[build]: https://travis-ci.org/4Catalyzer/flask-annex
+
 [pypi-badge]: https://img.shields.io/pypi/v/Flask-Annex.svg
 [pypi]: https://pypi.python.org/pypi/Flask-Annex
+
+[codecov-badge]: https://img.shields.io/codecov/c/github/4Catalyzer/flask-annex/master.svg
+[codecov]: https://codecov.io/gh/4Catalyzer/flask-annex

codecov.yml

@@ -0,0 +1 @@
+comment: off

flask_annex/file.py

@@ -1,9 +1,10 @@
 import errno
-import flask
 import glob
 import os
 import shutil
 
+import flask
+
 from .base import AnnexBase
 from .compat import string_types
 
@@ -62,7 +63,7 @@ def save_file(self, key, in_file):
             try:
                 os.makedirs(dir_name)
             except OSError as e:
-                if e.errno != errno.EEXIST:
+                if e.errno != errno.EEXIST:  # pragma: no cover
                     raise
 
         if isinstance(in_file, string_types):

flask_annex/s3.py

@@ -1,8 +1,8 @@
-from boto.s3.connection import S3Connection
-from boto.s3.key import Key
-import flask
 import mimetypes
 
+import boto3
+import flask
+
 from .base import AnnexBase
 from .compat import string_types
 
@@ -15,52 +15,72 @@
 
 class S3Annex(AnnexBase):
     def __init__(
-            self, bucket_name, access_key_id=None, secret_access_key=None,
-            url_expires_in=DEFAULT_URL_EXPIRES_IN):
-        self._connection = S3Connection(access_key_id, secret_access_key)
-        self._bucket = self._connection.get_bucket(
-            bucket_name, validate=False,
+        self,
+        bucket_name,
+        region=None,
+        access_key_id=None,
+        secret_access_key=None,
+        url_expires_in=DEFAULT_URL_EXPIRES_IN,
+    ):
+        self._client = boto3.client(
+            's3',
+            region,
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
         )
 
+        self._bucket_name = bucket_name
         self._url_expires_in = url_expires_in
 
-    def _get_s3_key(self, key):
-        return Key(self._bucket, key)
-
     def delete(self, key):
-        s3_key = self._get_s3_key(key)
-        s3_key.delete()
+        self._client.delete_object(Bucket=self._bucket_name, Key=key)
 
     def delete_many(self, keys):
-        self._bucket.delete_keys(keys)
+        self._client.delete_objects(
+            Bucket=self._bucket_name,
+            Delete={
+                'Objects': tuple({'Key': key} for key in keys),
+            },
+        )
 
     def get_file(self, key, out_file):
-        s3_key = self._get_s3_key(key)
-
         if isinstance(out_file, string_types):
-            s3_key.get_contents_to_filename(out_file)
+            self._client.download_file(self._bucket_name, key, out_file)
         else:
-            s3_key.get_contents_to_file(out_file)
+            self._client.download_fileobj(self._bucket_name, key, out_file)
 
     def list_keys(self, prefix):
-        return tuple(s3_key.name for s3_key in self._bucket.list(prefix))
+        response = self._client.list_objects_v2(
+            Bucket=self._bucket_name, Prefix=prefix,
+        )
+        if 'Contents' not in response:
+            return ()
+        return tuple(item['Key'] for item in response['Contents'])
 
     def save_file(self, key, in_file):
-        s3_key = self._get_s3_key(key)
-
         # Get the content type from the key, rather than letting Boto try to
         # figure it out from the file's name, which may be uninformative.
         content_type = mimetypes.guess_type(key)[0]
         if content_type:
-            headers = {'Content-Type': content_type}
+            extra_args = {'ContentType': content_type}
         else:
-            headers = None
+            extra_args = None
 
         if isinstance(in_file, string_types):
-            s3_key.set_contents_from_filename(in_file, headers=headers)
+            self._client.upload_file(
+                in_file, self._bucket_name, key, extra_args,
+            )
         else:
-            s3_key.set_contents_from_file(in_file, headers=headers)
+            self._client.upload_fileobj(
+                in_file, self._bucket_name, key, extra_args,
+            )
 
     def send_file(self, key):
-        s3_key = self._get_s3_key(key)
-        return flask.redirect(s3_key.generate_url(self._url_expires_in))
+        url = self._client.generate_presigned_url(
+            ClientMethod='get_object',
+            Params={
+                'Bucket': self._bucket_name,
+                'Key': key,
+            },
+        )
+        return flask.redirect(url)

setup.py

@@ -48,7 +48,7 @@ def run(self):
         'Flask >= 0.10',
     ),
     extras_require={
-        's3': ('boto >= 2.38.0',),
+        's3': ('boto3 >= 1.4.0',),
     },
     cmdclass={
         'clean': system('rm -rf build dist *.egg-info'),

tests/conftest.py

@@ -0,0 +1,12 @@
+from flask import Flask
+import pytest
+
+# -----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def app():
+    app = Flask(__name__)
+    app.config['TESTING'] = True
+
+    return app

tests/helpers.py

@@ -0,0 +1,64 @@
+from io import BytesIO
+
+import pytest
+
+# -----------------------------------------------------------------------------
+
+
+def assert_key_value(annex, key, value):
+    out_file = BytesIO()
+    annex.get_file(key, out_file)
+
+    out_file.seek(0)
+    assert out_file.read() == value
+
+
+# -----------------------------------------------------------------------------
+
+
+class AbstractTestAnnex(object):
+    @pytest.fixture
+    def annex(self, annex_base):
+        annex_base.save_file('foo/bar.txt', BytesIO(b'1\n'))
+        annex_base.save_file('foo/baz.json', BytesIO(b'2\n'))
+        return annex_base
+
+    def test_get_file(self, annex):
+        assert_key_value(annex, 'foo/bar.txt', b'1\n')
+
+    def test_get_filename(self, tmpdir, annex):
+        out_filename = tmpdir.join('out').strpath
+        annex.get_file('foo/bar.txt', out_filename)
+        assert open(out_filename).read() == '1\n'
+
+    def test_list_keys(self, annex):
+        assert sorted(annex.list_keys('foo/')) == [
+            'foo/bar.txt',
+            'foo/baz.json',
+        ]
+
+    def test_save_file(self, annex):
+        annex.save_file('qux/foo.txt', BytesIO(b'3\n'))
+        assert_key_value(annex, 'qux/foo.txt', b'3\n')
+
+    def test_save_filename(self, tmpdir, annex):
+        in_file = tmpdir.join('in')
+        in_file.write('4\n')
+
+        annex.save_file('qux/bar.txt', in_file.strpath)
+        assert_key_value(annex, 'qux/bar.txt', b'4\n')
+
+    def test_replace_file(self, annex):
+        assert_key_value(annex, 'foo/bar.txt', b'1\n')
+        annex.save_file('foo/bar.txt', BytesIO(b'5\n'))
+        assert_key_value(annex, 'foo/bar.txt', b'5\n')
+
+    def test_delete(self, annex):
+        assert annex.list_keys('foo/bar.txt')
+        annex.delete('foo/bar.txt')
+        assert not annex.list_keys('foo/bar.txt')
+
+    def test_delete_many(self, annex):
+        assert annex.list_keys('')
+        annex.delete_many(('foo/bar.txt', 'foo/baz.json'))
+        assert not annex.list_keys('')

tests/test_file.py

@@ -0,0 +1,43 @@
+from io import BytesIO
+
+import pytest
+
+from flask_annex import Annex
+
+from helpers import AbstractTestAnnex, assert_key_value
+
+# -----------------------------------------------------------------------------
+
+
+@pytest.fixture
+def file_annex_path(tmpdir):
+    return tmpdir.join('annex').strpath
+
+
+# -----------------------------------------------------------------------------
+
+
+class TestFileAnnex(AbstractTestAnnex):
+    @pytest.fixture
+    def annex_base(self, file_annex_path):
+        return Annex('file', root_path=file_annex_path)
+
+    def test_save_file_existing_dir(self, annex):
+        annex.save_file('foo/qux.txt', BytesIO(b'6\n'))
+        assert_key_value(annex, 'foo/qux.txt', b'6\n')
+
+    def test_send_file(self, app, annex):
+        with app.test_request_context():
+            response = annex.send_file('foo/baz.json')
+
+        assert response.status_code == 200
+        assert response.mimetype == 'application/json'
+
+
+class TestFileAnnexFromEnv(TestFileAnnex):
+    @pytest.fixture
+    def annex_base(self, monkeypatch, file_annex_path):
+        monkeypatch.setenv('FLASK_ANNEX_STORAGE', 'file')
+        monkeypatch.setenv('FLASK_ANNEX_FILE_ROOT_PATH', file_annex_path)
+
+        return Annex.from_env('FLASK_ANNEX')

tests/test_generic.py

@@ -0,0 +1,17 @@
+import pytest
+
+from flask_annex import Annex
+
+# -----------------------------------------------------------------------------
+
+
+def test_unknown_annex():
+    with pytest.raises(ValueError):
+        Annex('unknown')
+
+
+def test_unknown_annex_from_env(monkeypatch):
+    monkeypatch.setenv('FLASK_ANNEX_STORAGE', 'unknown')
+
+    with pytest.raises(ValueError):
+        Annex.from_env('FLASK_ANNEX')

tests/test_s3.py

@@ -0,0 +1,55 @@
+from io import BytesIO
+
+import pytest
+
+from flask_annex import Annex
+
+from helpers import AbstractTestAnnex, assert_key_value
+
+# -----------------------------------------------------------------------------
+
+try:
+    import boto3
+    from moto import mock_s3
+except ImportError:
+    pytestmark = pytest.mark.skipif(True, reason="S3 support not installed")
+
+# -----------------------------------------------------------------------------
+
+
+@pytest.yield_fixture
+def bucket_name():
+    with mock_s3():
+        bucket = boto3.resource('s3').Bucket('flask-annex')
+        bucket.create()
+
+        yield bucket.name
+
+
+# -----------------------------------------------------------------------------
+
+
+class TestS3Annex(AbstractTestAnnex):
+    @pytest.fixture
+    def annex_base(self, bucket_name):
+        return Annex('s3', bucket_name=bucket_name)
+
+    def test_save_file_unknown_type(self, annex):
+        annex.save_file('foo/qux', BytesIO(b'6\n'))
+        assert_key_value(annex, 'foo/qux', b'6\n')
+
+    def test_send_file(self, app, annex):
+        with app.test_request_context():
+            response = annex.send_file('foo/baz.json')
+
+        assert response.status_code == 302
+
+
+class TestS3AnnexFromEnv(TestS3Annex):
+    @pytest.fixture
+    def annex_base(self, monkeypatch, bucket_name):
+        monkeypatch.setenv('FLASK_ANNEX_STORAGE', 's3')
+        monkeypatch.setenv('FLASK_ANNEX_S3_BUCKET_NAME', bucket_name)
+        monkeypatch.setenv('FLASK_ANNEX_S3_REGION', 'us-east-1')
+
+        return Annex.from_env('FLASK_ANNEX')