Release beta 16

setup.py

@@ -2,15 +2,15 @@
 
 setup(
     name="hca-metadata-api",
-    version="1.0b15",
+    version="1.0b16",
     license='MIT',
     install_requires=[
         'dataclasses >= 0.6'
     ],
     # Not using tests_require because that installs the test requirements into .eggs, not the virtualenv
     extras_require={
         "dss": [
-            'hca == 4.4.1',
+            'hca == 5.2.0',
             'urllib3 >= 1.23'
         ],
         "examples": [

src/humancellatlas/data/metadata/helpers/dss.py

@@ -53,15 +53,27 @@ def download_bundle_metadata(client: DSSClient,
                        " guaranteed to stay in the code base in the future!")
 
     logger.debug("Getting bundle %s.%s from DSS.", uuid, version)
-    # noinspection PyUnresolvedReferences
-    response = client.get_bundle(uuid=uuid,
-                                 version=version,
-                                 replica=replica,
-                                 directurls=directurls,
-                                 presignedurls=presignedurls)
-    bundle = response['bundle']
-    manifest = bundle['files']
-    metadata_files = {f["name"]: f for f in manifest if f["indexed"]}
+    kwargs = dict(uuid=uuid,
+                  version=version,
+                  replica=replica,
+                  directurls=directurls,
+                  presignedurls=presignedurls)
+    url = None
+    manifest = []
+    while True:
+        # We can't use get_file.iterate because it only returns the `bundle.files` part of the response and swallows
+        # the `bundle.version`. See https://github.com/HumanCellAtlas/dcp-cli/issues/331
+        # noinspection PyUnresolvedReferences,PyProtectedMember
+        response = client.get_bundle._request(kwargs, url=url)
+        bundle = response.json()['bundle']
+        manifest.extend(bundle['files'])
+        try:
+            url = response.links['next']['url']
+        except KeyError:
+            break
+
+    metadata_files = {f['name']: f for f in manifest if f['indexed']}
+
     for f in metadata_files.values():
         content_type, _, _ = f['content-type'].partition(';')
         expected_content_type = 'application/json'

test/test.py

@@ -177,9 +177,10 @@ def test_bad_content(self):
 
     def test_bad_content_type(self):
         deployment, replica, uuid = 'staging', 'aws', 'df00a6fc-0015-4ae0-a1b7-d4b08af3c5a6'
-        client = Mock()
         file_uuid, file_version = 'b2216048-7eaa-45f4-8077-5a3fb4204953', '2018-09-20T232924.687620Z'
-        client.get_bundle.return_value = {
+        response = Mock()
+        response.links = {}
+        response.json.return_value = {
             'bundle': {
                 'files': [
                     {
@@ -192,6 +193,8 @@ def test_bad_content_type(self):
                 ]
             }
         }
+        client = Mock()
+        client.get_bundle._request.return_value = response
         with self.assertRaises(NotImplementedError) as cm:
             # noinspection PyTypeChecker
             download_bundle_metadata(client, replica, uuid)
@@ -215,6 +218,7 @@ def test_vx_primary_cs_bundle(self):
         A vx primary bundle with a cell_suspension as sequencing input
         """
         self._test_bundle(uuid='3e7c6f8e-334c-41fb-a1e5-ddd9fe70a0e2',
+                          version=None,
                           deployment='staging',
                           diseases={'glioblastoma'}),
 
@@ -427,6 +431,7 @@ def _assert_bundle(self, uuid, version, manifest, metadata_files,
         has_library_preps = library_construction_methods != set() or len(library_prep_protos) > 0
         self.assertEqual({LibraryPreparationProtocol} if has_library_preps else set(), library_prep_proto_types)
         self.assertEqual(library_construction_methods, {p.library_construction_method for p in library_prep_protos})
+        # noinspection PyDeprecation
         self.assertEqual(library_construction_methods, {p.library_construction_approach for p in library_prep_protos})
 
         if slice_thickness is not None:
@@ -494,6 +499,13 @@ def to_json(fqid):
 
         self.assertEqual({}, errors)
 
+    def test_large_bundle(self):
+        _, manifest, _ = download_bundle_metadata(client=dss_client('staging'),
+                                                  replica='aws',
+                                                  uuid='365c5f87-460a-41bc-a690-70ae6b5dba54',
+                                                  version='2018-10-17T092427.195428Z')
+        self.assertEqual(786, len(manifest))
+
     def test_analysis_protocol(self):
         uuid = 'ffee7f29-5c38-461a-8771-a68e20ec4a2e'
         version = '2019-02-02T065454.662896Z'
@@ -530,8 +542,10 @@ def assert_bundle():
             publication = project.publications.pop()
             title = 'Precursors of human CD4+ cytotoxic T lymphocytes identified by single-cell transcriptome analysis.'
             self.assertEqual(publication.title, title)
+            # noinspection PyDeprecation
             self.assertEqual(publication.title, publication.publication_title)
             self.assertEqual(publication.url, 'http://immunology.sciencemag.org/content/3/19/eaan8664.long')
+            # noinspection PyDeprecation
             self.assertEqual(publication.url, publication.publication_url)
             project_roles = {c.project_role for c in project.contributors}
             self.assertEqual(project_roles, {None, 'external curator', 'Human Cell Atlas wrangler'})
@@ -555,8 +569,15 @@ def assert_bundle():
             bundle = Bundle(uuid, version, manifest, metadata_files)
             project = bundle.projects[UUID('d96c2451-6e22-441f-a3e6-70fd0878bb1b')]
             self.assertEqual(len(project.contributors), 5)
-            expected_names = {'Sabina,,Kanton', 'Barbara,,Treutlein', 'J,Gray,Camp', 'Mallory,Ann,Freeberg', 'Zhisong,,He'}
+            expected_names = {
+                'Sabina,,Kanton',
+                'Barbara,,Treutlein',
+                'J,Gray,Camp',
+                'Mallory,Ann,Freeberg',
+                'Zhisong,,He'
+            }
             self.assertEqual({c.name for c in project.contributors}, expected_names)
+            # noinspection PyDeprecation
             self.assertEqual({c.contact_name for c in project.contributors}, expected_names)
 
         assert_bundle()
@@ -579,28 +600,35 @@ def assert_bundle():
                     self.assertEqual(file.format, 'fastq.gz')
                 if isinstance(file, SupplementaryFile):
                     self.assertEqual(file.format, 'pdf')
+                # noinspection PyDeprecation
                 self.assertEqual(file.format, file.file_format)
 
         assert_bundle()
 
-        for file in [metadata_files[f] for f in metadata_files if f.startswith('sequence_file_')
-                                                               or f.startswith('supplementary_file_')]:
-            self._rename_keys(file['file_core'], format='file_format')
+        for file_name, file_content in metadata_files.items():
+            if file_name.startswith('sequence_file_') or file_name.startswith('supplementary_file_'):
+                self._rename_keys(file_content['file_core'], format='file_format')
 
         assert_bundle()
 
     def test_link_destination_type(self):
         uuid = '6b498499-c5b4-452f-9ff9-2318dbb86000'
         version = '2019-01-03T163633.780215Z'
-        replica = 'aws'
-        deployment = 'prod'
-        manifest, metadata_files = self._load_bundle(uuid, version, replica, deployment)
+        manifest, metadata_files = self._load_bundle(uuid, version, replica='aws', deployment='prod')
 
         def assert_bundle():
             bundle = Bundle(uuid, version, manifest, metadata_files)
             destination_types = {link.destination_type for link in bundle.links}
-            expected_types = {'library_preparation_protocol', 'sequencing_protocol', 'dissociation_protocol',
-                              'differentiation_protocol', 'ipsc_induction_protocol', 'biomaterial', 'process', 'file'}
+            expected_types = {
+                'library_preparation_protocol',
+                'sequencing_protocol',
+                'dissociation_protocol',
+                'differentiation_protocol',
+                'ipsc_induction_protocol',
+                'biomaterial',
+                'process',
+                'file'
+            }
             self.assertEqual(destination_types, expected_types)
 
         assert_bundle()
@@ -611,6 +639,7 @@ def assert_bundle():
 
         assert_bundle()
 
+
 # noinspection PyUnusedLocal
 def load_tests(loader, tests, ignore):
     tests.addTests(doctest.DocTestSuite('humancellatlas.data.metadata.age_range'))