Release beta 22

.pycharm.style.xml

@@ -0,0 +1,37 @@
+<code_scheme name="Default" version="173">
+  <option name="FORMATTER_TAGS_ENABLED" value="true" />
+  <option name="ALIGN_MULTILINE_PARAMETERS_IN_CALLS" value="true" />
+  <option name="SOFT_MARGINS" value="80" />
+  <MarkdownNavigatorCodeStyleSettings>
+    <option name="USE_ACTUAL_CHAR_WIDTH" value="false" />
+    <option name="WRAP_ON_TYPING" value="1" />
+    <option name="DISABLE_WRAP_IN_SOFT_WRAP" value="false" />
+    <option name="KEEP_HARD_BREAKS" value="false" />
+    <option name="KEEP_TRAILING_SPACES" value="2" />
+    <option name="CODE_KEEP_TRAILING_SPACES" value="2" />
+    <option name="FORMAT_WITH_SOFT_WRAP" value="1" />
+    <option name="CODE_FENCE_MATCH_CLOSING_MARKER" value="true" />
+    <option name="RIGHT_MARGIN" value="80" />
+  </MarkdownNavigatorCodeStyleSettings>
+  <Python>
+    <option name="BLANK_LINES_BEFORE_FIRST_METHOD" value="1" />
+    <option name="OPTIMIZE_IMPORTS_SORT_NAMES_IN_FROM_IMPORTS" value="true" />
+    <option name="OPTIMIZE_IMPORTS_SORT_BY_TYPE_FIRST" value="false" />
+    <option name="OPTIMIZE_IMPORTS_JOIN_FROM_IMPORTS_WITH_SAME_SOURCE" value="true" />
+    <option name="FROM_IMPORT_WRAPPING" value="2" />
+    <option name="FROM_IMPORT_NEW_LINE_AFTER_LEFT_PARENTHESIS" value="true" />
+    <option name="FROM_IMPORT_NEW_LINE_BEFORE_RIGHT_PARENTHESIS" value="true" />
+    <option name="FROM_IMPORT_PARENTHESES_FORCE_IF_MULTILINE" value="true" />
+    <option name="FROM_IMPORT_TRAILING_COMMA_IF_MULTILINE" value="true" />
+  </Python>
+  <codeStyleSettings language="JSON">
+    <indentOptions>
+      <option name="INDENT_SIZE" value="4" />
+    </indentOptions>
+  </codeStyleSettings>
+  <codeStyleSettings language="Python">
+    <indentOptions>
+      <option name="CONTINUATION_INDENT_SIZE" value="4" />
+    </indentOptions>
+  </codeStyleSettings>
+</code_scheme>

.travis.yml

@@ -2,13 +2,24 @@ language: python
 
 cache: pip
 
+services:
+  - docker
+
 python:
   - 3.6
 
 install:
   - make travis_install
 
 script:
+  # Hack: The use of chrgp compensates for a quirk of Docker. The PyCharm image
+  # used by make format sets up a user called `developer` and assigns it UID
+  # 1000. Travis is running as UID 2000. An alternative would be to pass --user
+  # to `docker run` and bind-mount an /etc/passwd that maps that to `developer`.
+  # Currently I'm unclear why `make format` works unchanged on Docker
+  - sudo chgrp -R 1000 . && make format && sudo chgrp -R $(id -g) .
+  - make check_clean
+  - make pep8
   - make test
 
 after_success:

Makefile

@@ -8,6 +8,9 @@ ifneq ($(shell python -c "import sys; print(sys.version_info >= (3,6))"),True)
 $(error Looks like Python 3.6 is not installed or active in the current virtualenv)
 endif
 
+install_flake8:
+	pip install -U flake8==3.7.8
+
 install:
 	pip install -e .[dss,test,coverage,examples]
 
@@ -18,5 +21,22 @@ travis_install:
 test: install
 	coverage run -m unittest discover -vs test
 
+sources = src test
+
+pep8: install_flake8
+	flake8 --max-line-length=120 $(sources)
+
+format:
+	docker run \
+	    --rm \
+	    --volume $(CURDIR):/home/developer/metadata-api \
+	    --workdir /home/developer/metadata-api rycus86/pycharm:2019.2.3 \
+	    /opt/pycharm/bin/format.sh -r -settings .pycharm.style.xml -mask '*.py' $(sources)
+
+check_clean:
+	git diff --exit-code  && git diff --cached --exit-code
+
 examples: install
 	jupyter-notebook
+
+.PHONY: install_flake8 install travis_install test pep8 format check_clean examples

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="hca-metadata-api",
-    version="1.0b21",
+    version="1.0b22",
     license='MIT',
     install_requires=[
         'dataclasses >= 0.6'

src/humancellatlas/data/metadata/api.py

@@ -1,17 +1,37 @@
-from abc import ABC, abstractmethod
+from abc import (
+    ABC,
+    abstractmethod,
+)
 from collections import defaultdict
 from itertools import chain
-from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Type, TypeVar, Union
+from typing import (
+    Any,
+    Iterable,
+    List,
+    Mapping,
+    MutableMapping,
+    Optional,
+    Set,
+    Type,
+    TypeVar,
+    Union,
+)
 from uuid import UUID
 import warnings
 
-from dataclasses import dataclass, field
+from dataclasses import (
+    dataclass,
+    field,
+)
 
 from humancellatlas.data.metadata.age_range import AgeRange
 
 # A few helpful type aliases
 #
-from humancellatlas.data.metadata.lookup import lookup, LookupDefault
+from humancellatlas.data.metadata.lookup import (
+    lookup,
+    LookupDefault,
+)
 
 UUID4 = UUID
 AnyJSON2 = Union[str, int, float, bool, None, Mapping[str, Any], List[Any]]
@@ -556,6 +576,7 @@ def __init__(self, json: JSON):
         content = json.get('content', json)
         self.target = [ImagingTarget.from_json(target) for target in content['target']]
 
+
 @dataclass(init=False)
 class ImagingPreparationProtocol(Protocol):
     pass

src/humancellatlas/data/metadata/helpers/dss.py

@@ -73,19 +73,14 @@ def download_bundle_metadata(client: DSSClient,
                   replica=replica,
                   directurls=directurls,
                   presignedurls=presignedurls)
-    url = None
     manifest = []
-    while True:
-        # We can't use get_file.iterate because it only returns the `bundle.files` part of the response and swallows
-        # the `bundle.version`. See https://github.com/HumanCellAtlas/dcp-cli/issues/331
-        # noinspection PyUnresolvedReferences,PyProtectedMember
-        response = client.get_bundle._request(kwargs, url=url)
-        bundle = response.json()['bundle']
+
+    bundle = None
+    # noinspection PyUnresolvedReferences
+    for page in client.get_bundle.paginate(**kwargs):
+        bundle = page['bundle']
         manifest.extend(bundle['files'])
-        try:
-            url = response.links['next']['url']
-        except KeyError:
-            break
+    assert bundle is not None
 
     metadata_files = {f['name']: f for f in manifest if f['indexed']}
 

src/humancellatlas/data/metadata/helpers/json.py

@@ -1,7 +1,11 @@
 import copy
 from uuid import UUID
 
-from dataclasses import field, fields, is_dataclass
+from dataclasses import (
+    field,
+    fields,
+    is_dataclass,
+)
 
 from humancellatlas.data.metadata.api import Entity
 

src/humancellatlas/data/metadata/lookup.py

@@ -1,4 +1,8 @@
-from typing import TypeVar, Mapping, Union
+from typing import (
+    TypeVar,
+    Mapping,
+    Union,
+)
 from enum import Enum
 
 K = TypeVar('K')

test/humancellatlas/data/metadata/helpers/schema_examples.py

@@ -7,7 +7,7 @@
 from dcplib.checksumming_io import ChecksummingSink
 
 
-def download_example_bundle(repo, branch, path='/'):   # pragma: no cover (because of canning)
+def download_example_bundle(repo, branch, path='/'):  # pragma: no cover (because of canning)
     manifest = []
     metadata_files = {}
     if path.startswith('/') or not path.endswith('/'):

test/test.py

@@ -1,34 +1,45 @@
-from concurrent.futures import ThreadPoolExecutor, wait
+from concurrent.futures import (
+    ThreadPoolExecutor,
+    wait,
+)
 import doctest
 from itertools import chain
 from more_itertools import one
 import json
 import logging
 import os
 import re
-from unittest import TestCase, skip
+from unittest import (
+    TestCase,
+    skip,
+)
 from unittest.mock import Mock
 from uuid import UUID
 import warnings
 
 from atomicwrites import atomic_write
 
-from humancellatlas.data.metadata.api import (AgeRange,
-                                              Biomaterial,
-                                              Bundle,
-                                              DonorOrganism,
-                                              Project,
-                                              SequenceFile,
-                                              SpecimenFromOrganism,
-                                              CellLine,
-                                              CellSuspension,
-                                              AnalysisProtocol,
-                                              ImagingProtocol,
-                                              LibraryPreparationProtocol,
-                                              SequencingProtocol,
-                                              SupplementaryFile,
-                                              ImagedSpecimen)
-from humancellatlas.data.metadata.helpers.dss import download_bundle_metadata, dss_client
+from humancellatlas.data.metadata.api import (
+    AgeRange,
+    Biomaterial,
+    Bundle,
+    DonorOrganism,
+    Project,
+    SequenceFile,
+    SpecimenFromOrganism,
+    CellLine,
+    CellSuspension,
+    AnalysisProtocol,
+    ImagingProtocol,
+    LibraryPreparationProtocol,
+    SequencingProtocol,
+    SupplementaryFile,
+    ImagedSpecimen,
+)
+from humancellatlas.data.metadata.helpers.dss import (
+    download_bundle_metadata,
+    dss_client,
+)
 from humancellatlas.data.metadata.helpers.json import as_json
 from humancellatlas.data.metadata.helpers.schema_examples import download_example_bundle
 
@@ -169,24 +180,23 @@ def _load_bundle(self, uuid, version, replica='aws', deployment='prod'):
         return manifest, metadata_files
 
     def _mock_get_bundle(self, file_uuid, file_version, content_type):
-        response = Mock()
-        response.links = {}
-        response.json.return_value = {
-            'bundle': {
-                'version': '2018-09-20T232924.687620Z',
-                'files': [
-                    {
-                        'name': 'name.json',
-                        'uuid': file_uuid,
-                        'version': file_version,
-                        'indexed': True,
-                        'content-type': content_type
-                    }
-                ]
-            }
-        }
         client = Mock()
-        client.get_bundle._request.return_value = response
+        client.get_bundle.paginate.return_value = [
+            {
+                'bundle': {
+                    'version': '2018-09-20T232924.687620Z',
+                    'files': [
+                        {
+                            'name': 'name.json',
+                            'uuid': file_uuid,
+                            'version': file_version,
+                            'indexed': True,
+                            'content-type': content_type
+                        }
+                    ]
+                }
+            }
+        ]
         return client
 
     def test_bad_content(self):
@@ -210,8 +220,8 @@ def test_bad_content_type(self):
             # noinspection PyTypeChecker
             download_bundle_metadata(client, 'aws', uuid)
         self.assertEqual(cm.exception.args[0],
-                          f"Expecting file {file_uuid}.{file_version} "
-                          "to have content type 'application/json', not 'bad'")
+                         f"Expecting file {file_uuid}.{file_version} "
+                         "to have content type 'application/json', not 'bad'")
 
     def test_v5_bundle(self):
         """
@@ -575,6 +585,7 @@ def test_cell_line(self):
         self.assertEqual(cell_lines[0].biomaterial_id, 'cell_line_at_day_54')
         self.assertEqual(cell_lines[0].has_input_biomaterial, None)
         self.assertEqual(cell_lines[0].type, 'stem cell-derived')
+        # noinspection PyDeprecation
         self.assertEqual(cell_lines[0].type, cell_lines[0].cell_line_type)
         self.assertEqual(cell_lines[0].model_organ, 'brain')