From e267a6b0a76ba241fdc458615bc617899a30daaf Mon Sep 17 00:00:00 2001 From: Fritz Lekschas Date: Wed, 31 Oct 2018 11:05:09 -0400 Subject: [PATCH 01/18] Switch to settings based limitations --- fragments/utils.py | 24 +++++++++++++----------- higlass_server/settings.py | 7 +++++++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/fragments/utils.py b/fragments/utils.py index ce4d7924..1ef3dc9e 100644 --- a/fragments/utils.py +++ b/fragments/utils.py @@ -23,6 +23,8 @@ from hgtiles.geo import get_tile_pos_from_lng_lat +import higlass_server.settings as hss + from higlass_server.utils import getRdb from fragments.exceptions import SnippetTooLarge @@ -608,10 +610,6 @@ def get_frag_from_image_tiles( start2_rel = from_y - tile_start2_id * tile_size end2_rel = to_y - tile_start2_id * tile_size - # Make sure that image snippets are smaller or equal to 1024px - if end1_rel - start1_rel > 1024: raise SnippetTooLarge() - if end2_rel - start2_rel > 1024: raise SnippetTooLarge() - # Notice the shape: height x width x channel return np.array(im.crop((start1_rel, start2_rel, end1_rel, end2_rel))) @@ -679,8 +677,10 @@ def get_frag_by_loc_from_imtiles( tiles_y_range = range(tile_start2_id, tile_end2_id + 1) # Make sure that no more than 6 standard tiles (256px) are loaded. - if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge() - if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge() + if tile_size * len(tiles_x_range) > hss.SNIPPET_IMT_MAX_DATA_DIM: + raise SnippetTooLarge() + if tile_size * len(tiles_y_range) > hss.SNIPPET_IMT_MAX_DATA_DIM: + raise SnippetTooLarge() # Extract image tiles tiles = [] @@ -796,8 +796,10 @@ def get_frag_by_loc_from_osm( tiles_y_range = range(tile_start2_id, tile_end2_id + 1) # Make sure that no more than 6 standard tiles (256px) are loaded. - if tile_size * len(tiles_x_range) > 1536: raise SnippetTooLarge() - if tile_size * len(tiles_y_range) > 1536: raise SnippetTooLarge() + if tile_size * len(tiles_x_range) > hss.SNIPPET_OSM_MAX_DATA_DIM: + raise SnippetTooLarge() + if tile_size * len(tiles_y_range) > hss.SNIPPET_OSM_MAX_DATA_DIM: + raise SnippetTooLarge() # Extract image tiles tiles = [] @@ -1152,8 +1154,8 @@ def get_frag( abs_dim2 = height # Maximum width / height is 512 - if abs_dim1 > 512: raise SnippetTooLarge() - if abs_dim2 > 512: raise SnippetTooLarge() + if abs_dim1 > hss.SNIPPET_HIC_MAX_DATA_DIM: raise SnippetTooLarge() + if abs_dim2 > hss.SNIPPET_HIC_MAX_DATA_DIM: raise SnippetTooLarge() # Finally, adjust to negative values. # Since relative bin IDs are adjusted by the start this will lead to a @@ -1218,7 +1220,7 @@ def get_frag( # Assign 0 for now to avoid influencing the max values frag[low_quality_bins] = 0 - # Scale array if needed + # Scale fragment down if needed scaled = False scale_x = width / frag.shape[0] if frag.shape[0] > width or frag.shape[1] > height: diff --git a/higlass_server/settings.py b/higlass_server/settings.py index 9c6a3e0c..840da2cb 100644 --- a/higlass_server/settings.py +++ b/higlass_server/settings.py @@ -14,6 +14,7 @@ import os import os.path as op import slugid +import math from django.core.exceptions import ImproperlyConfigured @@ -277,6 +278,12 @@ def get_setting(name, default=None, settings=local_settings): UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True) PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True) +SNIPPET_HIC_MAX_OUT_DIM = get_setting('SNIPPET_HIC_MAX_OUT_DIM', math.inf) +SNIPPET_HIC_MAX_DATA_DIM = get_setting('SNIPPET_HIC_MAX_DATA_DIM', math.inf) +SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', math.inf) +SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', math.inf) +SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', math.inf) + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ From ad5c143f5198d30cefa1adfd5acb877e52322b17 Mon Sep 17 00:00:00 2001 From: Fritz Lekschas Date: Wed, 31 Oct 2018 11:05:33 -0400 Subject: [PATCH 02/18] If zoom-level == -1 use most efficient zoom level to pull out snippet --- fragments/views.py | 49 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/fragments/views.py b/fragments/views.py index 5ff83d8d..61fef896 100755 --- a/fragments/views.py +++ b/fragments/views.py @@ -11,6 +11,8 @@ except: import pickle +import higlass_server.settings as hss + from rest_framework.authentication import BasicAuthentication from .drf_disable_csrf import CsrfExemptSessionAuthentication from io import BytesIO @@ -39,6 +41,10 @@ from higlass_server.utils import getRdb from fragments.exceptions import SnippetTooLarge +import h5py + +from math import floor, log + rdb = getRdb() logger = logging.getLogger(__name__) @@ -232,7 +238,8 @@ def get_fragments_by_loci(request): encoding = params['encoding'] representatives = params['representatives'] - tileset_idx = 6 if len(loci) and len(loci[0]) > 7 else 4 + is_cool = len(loci) and len(loci[0]) > 7 + tileset_idx = 6 if is_cool else 4 zoom_level_idx = tileset_idx + 1 filetype = None @@ -286,12 +293,7 @@ def get_fragments_by_loci(request): 'error': 'Tileset not specified', }, status=400) - if tileset_file not in loci_lists: - loci_lists[tileset_file] = {} - - if locus[zoom_level_idx] not in loci_lists[tileset_file]: - loci_lists[tileset_file][locus[zoom_level_idx]] = [] - + # Get the dimensions of the snippets (i.e., width and height in px) inset_dim = ( locus[zoom_level_idx + 1] if ( @@ -300,10 +302,41 @@ def get_fragments_by_loci(request): ) else 0 ) + out_dim = inset_dim | dims + + # Make sure out dim (in pixel) is not too large + if is_cool and out_dim > hss.SNIPPET_HIC_MAX_OUT_DIM: + raise SnippetTooLarge() + if not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM: + raise SnippetTooLarge() + + if tileset_file not in loci_lists: + loci_lists[tileset_file] = {} + + if is_cool: + with h5py.File(tileset_file, 'r') as f: + # get base resolution of cooler file + max_zoom = f.attrs['max-zoom'] + bin_size = int(f[str(max_zoom)].attrs['bin-size']) + else: + bin_size = 1 + + # Get max abs dim in base pairs + max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4]) + + # Find clostest zoom level if `zoomout_level < 0` + zoomout_level = ( + locus[zoom_level_idx] + if locus[zoom_level_idx] >= 0 + else floor(log((max_abs_dim / bin_size) / out_dim, 2)) + ) + + if zoomout_level not in loci_lists[tileset_file]: + loci_lists[tileset_file][zoomout_level] = [] locus_id = '.'.join(map(str, locus)) - loci_lists[tileset_file][locus[zoom_level_idx]].append( + loci_lists[tileset_file][zoomout_level].append( locus[0:tileset_idx] + [i, inset_dim, locus_id] ) loci_ids.append(locus_id) From a257f3acb9e7f9d29076a0436f3d80ac9db5a34c Mon Sep 17 00:00:00 2001 From: Fritz Lekschas Date: Wed, 31 Oct 2018 11:08:58 -0400 Subject: [PATCH 03/18] Update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f2a662d..965a0dad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v1.7.? (????-??-??) + +- Snippets API now allows limiting the size of snippets via `config.json` + v1.7.3 (2018-07-12) - Return datatype along with tileset info From 19ffbd71f18d4486cb4ec8a8b8fc67beec91f745 Mon Sep 17 00:00:00 2001 From: Fritz Lekschas Date: Wed, 31 Oct 2018 12:03:43 -0400 Subject: [PATCH 04/18] Adjust to Pete's comments --- fragments/utils.py | 4 ++-- fragments/views.py | 11 +++-------- higlass_server/settings.py | 4 ++-- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/fragments/utils.py b/fragments/utils.py index 1ef3dc9e..28c9c120 100644 --- a/fragments/utils.py +++ b/fragments/utils.py @@ -1154,8 +1154,8 @@ def get_frag( abs_dim2 = height # Maximum width / height is 512 - if abs_dim1 > hss.SNIPPET_HIC_MAX_DATA_DIM: raise SnippetTooLarge() - if abs_dim2 > hss.SNIPPET_HIC_MAX_DATA_DIM: raise SnippetTooLarge() + if abs_dim1 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() + if abs_dim2 > hss.SNIPPET_MAT_MAX_DATA_DIM: raise SnippetTooLarge() # Finally, adjust to negative values. # Since relative bin IDs are adjusted by the start this will lead to a diff --git a/fragments/views.py b/fragments/views.py index 61fef896..8dc3f77e 100755 --- a/fragments/views.py +++ b/fragments/views.py @@ -238,6 +238,7 @@ def get_fragments_by_loci(request): encoding = params['encoding'] representatives = params['representatives'] + # Check if requesting a snippet from a `.cool` cooler file is_cool = len(loci) and len(loci[0]) > 7 tileset_idx = 6 if is_cool else 4 zoom_level_idx = tileset_idx + 1 @@ -305,7 +306,7 @@ def get_fragments_by_loci(request): out_dim = inset_dim | dims # Make sure out dim (in pixel) is not too large - if is_cool and out_dim > hss.SNIPPET_HIC_MAX_OUT_DIM: + if is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM: raise SnippetTooLarge() if not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM: raise SnippetTooLarge() @@ -324,7 +325,7 @@ def get_fragments_by_loci(request): # Get max abs dim in base pairs max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4]) - # Find clostest zoom level if `zoomout_level < 0` + # Find closest zoom level if `zoomout_level < 0` zoomout_level = ( locus[zoom_level_idx] if locus[zoom_level_idx] >= 0 @@ -442,12 +443,6 @@ def get_fragments_by_loci(request): data_types[idx] = 'matrix' - except SnippetTooLarge as ex: - raise - return JsonResponse({ - 'error': 'Requested fragment too large. Max is 1024x1024! Behave!', - 'error_message': str(ex) - }, status=400) except Exception as ex: raise return JsonResponse({ diff --git a/higlass_server/settings.py b/higlass_server/settings.py index 840da2cb..c76b2a73 100644 --- a/higlass_server/settings.py +++ b/higlass_server/settings.py @@ -278,8 +278,8 @@ def get_setting(name, default=None, settings=local_settings): UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True) PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True) -SNIPPET_HIC_MAX_OUT_DIM = get_setting('SNIPPET_HIC_MAX_OUT_DIM', math.inf) -SNIPPET_HIC_MAX_DATA_DIM = get_setting('SNIPPET_HIC_MAX_DATA_DIM', math.inf) +SNIPPET_MAT_MAX_OUT_DIM = get_setting('SNIPPET_MAT_MAX_OUT_DIM', math.inf) +SNIPPET_MAT_MAX_DATA_DIM = get_setting('SNIPPET_MAT_MAX_DATA_DIM', math.inf) SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', math.inf) SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', math.inf) SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', math.inf) From 5563d6fe18d8a2eff41510cea7212eae94f71ffe Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Sat, 1 Dec 2018 16:57:41 -0500 Subject: [PATCH 05/18] Retrieve project name in TilesetSerializer --- tilesets/serializers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tilesets/serializers.py b/tilesets/serializers.py index c3083329..a6367e6c 100644 --- a/tilesets/serializers.py +++ b/tilesets/serializers.py @@ -42,6 +42,13 @@ class TilesetSerializer(serializers.ModelSerializer): slug_field='uuid', allow_null=True, required=False) + project_name = serializers.SerializerMethodField('retrieve_project_name') + + def retrieve_project_name(self, obj): + if obj.project is None: + return '' + + return obj.project.name class Meta: owner = serializers.ReadOnlyField(source='owner.username') @@ -56,6 +63,7 @@ class Meta: 'coordSystem2', 'created', 'project', + 'project_name', 'description', 'private', ) @@ -66,7 +74,6 @@ class UserFacingTilesetSerializer(TilesetSerializer): project_name = serializers.SerializerMethodField('retrieve_project_name') project_owner = serializers.SerializerMethodField('retrieve_project_owner') - def retrieve_project_name(self, obj): if obj.project is None: return '' From baba6bca12dfa558d1df1e19572ffb755dee836b Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 8 Jan 2019 10:01:47 -0500 Subject: [PATCH 06/18] Bumped django version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1169ff16..bd116dea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ cooler==0.7.10 django-cors-headers django-guardian django-rest-swagger -django==2.0.9 +django==2.1.5 djangorestframework==3.7.3 h5py==2.6.0 numba==0.37.0 From 9668c901637a6db3e6b58f04299c5e1c72ed5509 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Sat, 19 Jan 2019 09:49:32 -0800 Subject: [PATCH 07/18] Bumped clodius version number --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bd116dea..9f05de6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,4 @@ scikit-learn==0.19.1 slugid==1.0.7 bumpversion==0.5.3 redis==2.10.5 -clodius>=0.10.0.rc2 +clodius>=0.10.0 From c594ee5204b3f593649baf041c8b29d946927933 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 5 Feb 2019 09:19:12 -0500 Subject: [PATCH 08/18] Added a slash to the end of the static url --- higlass_server/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/higlass_server/settings.py b/higlass_server/settings.py index e2dd8ef0..4abe81c6 100644 --- a/higlass_server/settings.py +++ b/higlass_server/settings.py @@ -314,7 +314,7 @@ def get_setting(name, default=None, settings=local_settings): LOGIN_REDIRECT_URL = os.environ['APP_BASEPATH'] LOGOUT_REDIRECT_URL = os.environ['APP_BASEPATH'] - STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static') + STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static') + "/" ADMIN_URL = r'^admin/' From 9520547e2389857e52b576d98365cf1847ec9f25 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 5 Feb 2019 09:19:59 -0500 Subject: [PATCH 09/18] Updated the CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 965a0dad..5e683016 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +v1.9.2 + +- Fixed STATIC_URL settings must end with a slash bug + +v1.9.1 + +- Added support for the APP_BASEPATH setting + v1.7.? (????-??-??) - Snippets API now allows limiting the size of snippets via `config.json` From 7a0828da9b98b13bf8bb15ac14dacafac071586c Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Sun, 10 Feb 2019 22:40:48 -0500 Subject: [PATCH 10/18] Bumped the clodius version --- requirements.txt | 2 +- tilesets/generate_tiles.py | 5 +- tilesets/multivec_tiles.py | 244 ------------------------------------- tilesets/views.py | 1 - unit_tests.sh | 1 + 5 files changed, 5 insertions(+), 248 deletions(-) delete mode 100644 tilesets/multivec_tiles.py diff --git a/requirements.txt b/requirements.txt index 9900ea12..bdd36509 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,5 +20,5 @@ scikit-learn==0.19.1 slugid==1.0.7 bumpversion==0.5.3 redis==2.10.5 -clodius>=0.10.1 +clodius>=0.10.3 simple-httpfs>=0.1.3 diff --git a/tilesets/generate_tiles.py b/tilesets/generate_tiles.py index ea548387..9c108df2 100644 --- a/tilesets/generate_tiles.py +++ b/tilesets/generate_tiles.py @@ -19,7 +19,8 @@ import tempfile import tilesets.models as tm import tilesets.chromsizes as tcs -import tilesets.multivec_tiles as tmt + +import clodius.tiles.multivec as ctmu import higlass_server.settings as hss @@ -504,7 +505,7 @@ def generate_tiles(tileset_tile_ids): return generate_1d_tiles( tileset.datafile.path, tile_ids, - tmt.get_single_tile) + ctmu.get_single_tile) elif tileset.filetype == 'imtiles': return hgim.get_tiles(tileset.datafile.path, tile_ids, raw) else: diff --git a/tilesets/multivec_tiles.py b/tilesets/multivec_tiles.py deleted file mode 100644 index 60e5672d..00000000 --- a/tilesets/multivec_tiles.py +++ /dev/null @@ -1,244 +0,0 @@ -import h5py -import numpy as np -import time - -def abs2genomic(chromsizes, start_pos, end_pos): - ''' - Convert absolute genomic sizes to genomic - - Parameters: - ----------- - chromsizes: [1000,...] - An array of the lengths of the chromosomes - start_pos: int - The starting genomic position - end_pos: int - The ending genomic position - ''' - abs_chrom_offsets = np.r_[0, np.cumsum(chromsizes)] - cid_lo, cid_hi = np.searchsorted(abs_chrom_offsets, - [start_pos, end_pos], - side='right') - 1 - rel_pos_lo = start_pos - abs_chrom_offsets[cid_lo] - rel_pos_hi = end_pos - abs_chrom_offsets[cid_hi] - start = rel_pos_lo - for cid in range(cid_lo, cid_hi): - yield cid, start, chromsizes[cid] - start = 0 - yield cid_hi, start, rel_pos_hi - -def get_tileset_info(filename): - ''' - Return some information about this tileset that will - help render it in on the client. - Parameters - ---------- - filename: str - The filename of the h5py file containing the tileset info. - Returns - ------- - tileset_info: {} - A dictionary containing the information describing - this dataset - ''' - t1 = time.time() - f = h5py.File(filename, 'r') - t2 = time.time() - # a sorted list of resolutions, lowest to highest - # awkward to write because a the numbers representing resolution - # are datapoints / pixel so lower resolution is actually a higher - # number - resolutions = sorted([int(r) for r in f['resolutions'].keys()])[::-1] - - # the "leftmost" datapoint position - # an array because higlass can display multi-dimensional - # data - min_pos = [0] - max_pos = [int(sum(f['chroms']['length'][:]))] - - # the "rightmost" datapoint position - # max_pos = [len(f['resolutions']['values'][str(resolutions[-1])])] - tile_size = int(f['info'].attrs['tile-size']) - first_chrom = f['chroms']['name'][0] - - shape = list(f['resolutions'][str(resolutions[0])]['values'][first_chrom].shape) - shape[0] = tile_size - - # print("tileset info time:", t3 - t2) - - tileset_info = { - 'resolutions': resolutions, - 'min_pos': min_pos, - 'max_pos': max_pos, - 'tile_size': tile_size, - 'shape': shape - } - - # print("hi:", list(f['resolutions'][str(resolutions[0])].attrs.keys())) - if 'row_infos' in f['resolutions'][str(resolutions[0])].attrs: - row_infos = f['resolutions'][str(resolutions[0])].attrs['row_infos'] - tileset_info['row_infos'] = [r.decode('utf8') for r in row_infos] - - f.close() - t3 = time.time() - - return tileset_info - - -def get_single_tile(filename, tile_pos): - ''' - Retrieve a single multivec tile from a multires file - Parameters - ---------- - filename: string - The multires file containing the multivec data - tile_pos: (z, x) - The zoom level and position of this tile - ''' - t1 = time.time() - tileset_info = get_tileset_info(filename) - - t15 = time.time() - - f = h5py.File(filename, 'r') - - # print('tileset_info', tileset_info) - t2 = time.time() - # which resolution does this zoom level correspond to? - resolution = tileset_info['resolutions'][tile_pos[0]] - tile_size = tileset_info['tile_size'] - - # where in the data does the tile start and end - tile_start = tile_pos[1] * tile_size * resolution - tile_end = tile_start + tile_size * resolution - - chromsizes = list(zip(f['chroms']['name'], f['chroms']['length'])) - - #dense = f['resolutions'][str(resolution)][tile_start:tile_end] - dense = get_tile(f, chromsizes, resolution, tile_start, tile_end, tileset_info['shape']) - #print("dense.shape", dense.shape) - - if len(dense) < tileset_info['tile_size']: - # if there aren't enough rows to fill this tile, add some zeros - dense = np.vstack([dense, np.zeros((tileset_info['tile_size'] - len(dense), - tileset_info['shape'][1]))]) - - f.close() - t3 = time.time() - - # print("single time time: {:.2f} (tileset info: {:.2f}, open time: {:.2f})".format(t3 - t1, t15 - t1, t2 - t15)) - - return dense.T - -def get_tile(f, chromsizes, resolution, start_pos, end_pos, shape): - ''' - Get the tile value given the start and end positions and - chromosome positions. - - Drop bins at the ends of chromosomes if those bins aren't - full. - - Parameters: - ----------- - f: h5py.File - An hdf5 file containing the data - chromsizes: [('chr1', 1000), ....] - An array listing the chromosome sizes - resolution: int - The size of each bin, except for the last bin in each - chromosome. - start_pos: int - The start_position of the interval to return - end_pos: int - The end position of the interval to return - - Returns - ------- - return_vals: [...] - A subset of the original genome-wide values containing - the values for the portion of the genome that is visible. - ''' - binsize = resolution - # print('binsize:', binsize) - # print('start_pos:', start_pos, 'end_pos:', end_pos) - # print("length:", end_pos - start_pos) - # print('shape:', shape) - - t0 = time.time() - arrays = [] - count = 0 - - # keep track of how much data has been returned in bins - current_binned_data_position = 0 - current_data_position = 0 - - num_added = 0 - total_length = 0 - - for cid, start, end in abs2genomic([c[1] for c in chromsizes], start_pos, end_pos): - n_bins = int(np.ceil((end - start) / binsize)) - total_length += end - start - - try: - t1 = time.time() - - chrom = chromsizes[cid][0] - clen = chromsizes[cid][1] - - current_data_position += end - start - - count += 1 - - start_pos = start // binsize - end_pos = end // binsize - - # print('current_data_position:', current_data_position) - # print('current_binned_data_position:', current_binned_data_position) - # print('binsize:', binsize, 'resolution:', resolution) - - if start_pos == end_pos: - if current_data_position - current_binned_data_position > 0: - # adding this data as a single bin even though it's not large - # enough to cover one bin - #print('catching up') - end_pos += 1 - else: - #print('data smaller than the bin size', start, end, binsize) - continue - - if chrom not in f['resolutions'][str(resolution)]['values']: - continue - - # print('values:', f['resolutions'][str(resolution)]['values'][chrom][:]) - x = f['resolutions'][str(resolution)]['values'][chrom][start_pos:end_pos] - current_binned_data_position += binsize * (end_pos - start_pos) - - # print("x:", x.shape) - - # drop the very last bin if it is smaller than the binsize - if len(x) > 1 and end == clen and clen % binsize != 0: - #print("dropping") - x = x[:-1] - - if len(x): - num_added += 1 - # print('num_added:', num_added, 'x:', sum(x[0])) - - t2 = time.time() - - # print("time to fetch {}: {}".format(chrom, t2 - t1)) - except IndexError: - # beyond the range of the available chromosomes - # probably means we've requested a range of absolute - # coordinates that stretch beyond the end of the genome - # print('zeroes') - x = np.zeros((n_bins, shape[1])) - - arrays.append(x) - - # print("arrays:", arrays[0]) - # print("total_length:", total_length) - t3 = time.time() - # print("total fetch time:", t3 - t0) - - return np.concatenate(arrays) diff --git a/tilesets/views.py b/tilesets/views.py index 76e27bc8..1cd2f482 100644 --- a/tilesets/views.py +++ b/tilesets/views.py @@ -24,7 +24,6 @@ import tilesets.chromsizes as tcs import tilesets.generate_tiles as tgt -import tilesets.multivec_tiles as tmt import clodius.tiles.cooler as hgco import clodius.tiles.bigwig as hgbi diff --git a/unit_tests.sh b/unit_tests.sh index b1823245..7a8e4b99 100755 --- a/unit_tests.sh +++ b/unit_tests.sh @@ -7,5 +7,6 @@ simple-httpfs.py media/http simple-httpfs.py media/https python manage.py test tilesets.tests.FileUploadTest --failfast +python manage.py test tilesets.tests.MultivecTests --failfast #python manage.py test tilesets --failfast From 174dff09682611267bb42c99b720dfb81421d25d Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 12 Mar 2019 23:16:14 -0400 Subject: [PATCH 11/18] Updated CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e683016..455f5bac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v1.10.0 + +- Added support for mrmatrix files +- Small bug fix for 500 available-chrom-sizes + v1.9.2 - Fixed STATIC_URL settings must end with a slash bug From 8c7248a6cf3bd73f769f5713c0f469d40251bb10 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 12 Mar 2019 23:32:45 -0400 Subject: [PATCH 12/18] Bumped Dockerfile miniconda version --- docker-context/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-context/Dockerfile b/docker-context/Dockerfile index 4811117b..a1545c0e 100644 --- a/docker-context/Dockerfile +++ b/docker-context/Dockerfile @@ -1,4 +1,4 @@ -FROM continuumio/miniconda3:4.3.14 +FROM continuumio/miniconda3:4.5.12 RUN apt-get update && apt-get install -y \ gcc=4:4.9.2-2 \ From 03cb234aa18c544c2bd7141de69b6e11580c0264 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Tue, 12 Mar 2019 23:40:15 -0400 Subject: [PATCH 13/18] Unpinned gcc --- docker-context/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-context/Dockerfile b/docker-context/Dockerfile index a1545c0e..404fb8fe 100644 --- a/docker-context/Dockerfile +++ b/docker-context/Dockerfile @@ -1,7 +1,7 @@ FROM continuumio/miniconda3:4.5.12 RUN apt-get update && apt-get install -y \ - gcc=4:4.9.2-2 \ + gcc \ nginx-full \ supervisor \ unzip=6.0-16+deb8u2 \ From d5b5658490daa0e5da881d2b28621ed4760cc91e Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Wed, 13 Mar 2019 08:25:52 -0400 Subject: [PATCH 14/18] Unpinned all packages in Dockerfile --- docker-context/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-context/Dockerfile b/docker-context/Dockerfile index 404fb8fe..19d79adb 100644 --- a/docker-context/Dockerfile +++ b/docker-context/Dockerfile @@ -4,9 +4,9 @@ RUN apt-get update && apt-get install -y \ gcc \ nginx-full \ supervisor \ - unzip=6.0-16+deb8u2 \ + unzip \ uwsgi-plugin-python3 \ - zlib1g-dev=1:1.2.8.dfsg-2+b1 \ + zlib1g-dev \ libcurl4-openssl-dev \ g++ \ vim \ From bc8c581b41b483811dc11143d5572a08ae278b06 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Wed, 13 Mar 2019 11:34:11 -0400 Subject: [PATCH 15/18] Check to make sure project's owner is not none before returning username --- tilesets/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tilesets/serializers.py b/tilesets/serializers.py index a6367e6c..428ecb9b 100644 --- a/tilesets/serializers.py +++ b/tilesets/serializers.py @@ -84,6 +84,9 @@ def retrieve_project_owner(self, obj): if obj.project is None: return '' + if obj.project.owner is None: + return '' + return obj.project.owner.username class Meta: From 5a34912cc6f806f026aaa372173b2dee92ba5c19 Mon Sep 17 00:00:00 2001 From: Peter Kerpedjiev Date: Sat, 16 Mar 2019 09:35:26 -0400 Subject: [PATCH 16/18] Added project to admin interface and removed coordSystem2 as a required field for tilesets --- CHANGELOG.md | 9 +++++++++ tilesets/admin.py | 10 ++++++++++ tilesets/models.py | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 455f5bac..7df95082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +v1.10.2 + +- Added project to the admin interface +- coordSystem2 is no longer a required field + +v1.10.1 + +- Check to make sure project's owner is not None before returning username + v1.10.0 - Added support for mrmatrix files diff --git a/tilesets/admin.py b/tilesets/admin.py index 4156e11e..186f6767 100644 --- a/tilesets/admin.py +++ b/tilesets/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin from tilesets.models import Tileset from tilesets.models import ViewConf +from tilesets.models import Project # Register your models here. @@ -25,7 +26,16 @@ class ViewConfAdmin(admin.ModelAdmin): 'uuid', 'higlassVersion', ] + +class ProjectConfAdmin(admin.ModelAdmin): + list_display = [ + 'created', + 'uuid', + 'name', + 'description', + ] admin.site.register(Tileset, TilesetAdmin) admin.site.register(ViewConf, ViewConfAdmin) +admin.site.register(Project, ProjectConfAdmin) diff --git a/tilesets/models.py b/tilesets/models.py index aad6f663..6fa575a8 100644 --- a/tilesets/models.py +++ b/tilesets/models.py @@ -24,7 +24,7 @@ def __str__(self): return "Viewconf [uuid: {}]".format(self.uuid) def decoded_slugid(): - return slugid.nice().decode('utf-8') + return slugid.nice() class Project(models.Model): created = models.DateTimeField(auto_now_add=True) @@ -60,7 +60,7 @@ class Tileset(models.Model): description = models.TextField(blank=True) coordSystem = models.TextField() - coordSystem2 = models.TextField(default='') + coordSystem2 = models.TextField(default='', blank=True) temporary = models.BooleanField(default=False) owner = models.ForeignKey( From 82cc2b2068e1eee5793c1d0f8321bbbff3bbec87 Mon Sep 17 00:00:00 2001 From: Nezar Date: Tue, 7 May 2019 12:48:07 -0400 Subject: [PATCH 17/18] Delete backing media file on removal of a tileset --- tilesets/models.py | 42 +++++++++++++++++++++++++++++------------- tilesets/views.py | 10 ++++++---- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/tilesets/models.py b/tilesets/models.py index 6fa575a8..52dbbfb8 100644 --- a/tilesets/models.py +++ b/tilesets/models.py @@ -5,6 +5,12 @@ import slugid from django.db import models +from django.db.models.signals import post_delete +from django.dispatch import receiver + + +def decoded_slugid(): + return slugid.nice() class ViewConf(models.Model): @@ -23,40 +29,42 @@ def __str__(self): ''' return "Viewconf [uuid: {}]".format(self.uuid) -def decoded_slugid(): - return slugid.nice() class Project(models.Model): created = models.DateTimeField(auto_now_add=True) last_viewed_time = models.DateTimeField(default=django.utils.timezone.now) - owner = models.ForeignKey(dcam.User, on_delete=models.CASCADE, blank=True, null=True) + owner = models.ForeignKey( + dcam.User, on_delete=models.CASCADE, blank=True, null=True) name = models.TextField(unique=True) description = models.TextField(blank=True) - uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) + uuid = models.CharField(max_length=100, unique=True, + default=decoded_slugid) private = models.BooleanField(default=False) class Meta: ordering = ('created',) permissions = (('read', "Read permission"), - ('write', 'Modify tileset'), - ('admin', 'Administrator priviliges'), - ) + ('write', 'Modify tileset'), + ('admin', 'Administrator priviliges'), + ) def __str__(self): return "Project [name: " + self.name + "]" + class Tileset(models.Model): created = models.DateTimeField(auto_now_add=True) - uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) + uuid = models.CharField(max_length=100, unique=True, + default=decoded_slugid) # processed_file = models.TextField() datafile = models.FileField(upload_to='uploads') filetype = models.TextField() datatype = models.TextField(default='unknown', blank=True, null=True) project = models.ForeignKey(Project, on_delete=models.CASCADE, - blank=True, null=True) + blank=True, null=True) description = models.TextField(blank=True) coordSystem = models.TextField() @@ -73,13 +81,21 @@ class Tileset(models.Model): class Meta: ordering = ('created',) permissions = (('read', "Read permission"), - ('write', 'Modify tileset'), - ('admin', 'Administrator priviliges'), - ) + ('write', 'Modify tileset'), + ('admin', 'Administrator priviliges'), + ) def __str__(self): ''' Get a string representation of this model. Hopefully useful for the admin interface. ''' - return "Tileset [name: {}] [ft: {}] [uuid: {}]".format(self.name, self.filetype, self.uuid) + return "Tileset [name: {}] [ft: {}] [uuid: {}]".format( + self.name, self.filetype, self.uuid) + + +@receiver(post_delete, sender=Tileset) +def tileset_on_delete(sender, instance, **kwargs): + + if not instance.datafile.name.endswith('..'): + instance.datafile.delete(False) diff --git a/tilesets/views.py b/tilesets/views.py index 34025c0b..cf70687d 100644 --- a/tilesets/views.py +++ b/tilesets/views.py @@ -758,12 +758,14 @@ def destroy(self, request, *args, **kwargs): return JsonResponse({'error': 'The uuid parameter is undefined'}, status=400) try: instance = self.get_object() - self.perform_destroy(instance) - filename = instance.datafile.name - filepath = op.join(hss.MEDIA_ROOT, filename) + + filepath = op.join(hss.MEDIA_ROOT, instance.datafile.name) if not op.isfile(filepath): return JsonResponse({'error': 'Unable to locate tileset media file for deletion: {}'.format(filepath)}, status=500) - os.remove(filepath) + + # model's post-destroy handler does the actual file deletion + self.perform_destroy(instance) + except dh.Http404: return JsonResponse({'error': 'Unable to locate tileset instance for uuid: {}'.format(uuid)}, status=404) except dbm.ProtectedError as dbpe: From 1d3a7cd5fd86131e54412890fbae2576985ced7d Mon Sep 17 00:00:00 2001 From: Nezar Date: Tue, 7 May 2019 14:00:33 -0400 Subject: [PATCH 18/18] Add test for Tileset model post-delete hook --- tilesets/tests.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tilesets/tests.py b/tilesets/tests.py index 0a6859cc..af2e33b2 100644 --- a/tilesets/tests.py +++ b/tilesets/tests.py @@ -51,8 +51,8 @@ def media_file_exists(filename): ------- The return value. True for success, False otherwise. ''' - return False if not op.exists(media_file(filename)) else True - + return False if not op.exists(media_file(filename)) else True + def add_file(filename, sub_dir='uploads/data'): ''' @@ -468,8 +468,8 @@ def test_to_string(self): ) upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') self.cooler = tm.Tileset.objects.create( - datafile=dcfu.SimpleUploadedFile(upload_file.name, - upload_file.read()), + datafile=dcfu.SimpleUploadedFile( + upload_file.name, upload_file.read()), filetype='cooler', owner=self.user1, uuid='x1x' @@ -478,6 +478,25 @@ def test_to_string(self): cooler_string = str(self.cooler) self.assertTrue(cooler_string.find("name") > 0) + def test_destroy_deletes_file(self): + self.user1 = dcam.User.objects.create_user( + username='user1', password='pass' + ) + upload_file = open('data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'rb') + ts = tm.Tileset.objects.create( + datafile=dcfu.SimpleUploadedFile( + upload_file.name, upload_file.read()), + filetype='cooler', + owner=self.user1, + uuid='x2x' + ) + filepath = op.join(hss.MEDIA_ROOT, ts.datafile.name) + self.assertTrue(op.exists(filepath)) + + ts.delete() + self.assertFalse(op.exists(filepath)) + + class UnknownTilesetTypeTest(dt.TestCase): def setUp(self): self.user1 = dcam.User.objects.create_user( @@ -643,11 +662,11 @@ def test_permissions(self): assert(response.status_code == 201) ret = json.loads(response.content.decode('utf-8')) - + # update media filename for whatever name the server ended up using (i.e., in case of duplicates, a random suffix is added) assert('datafile' in ret) fname = op.basename(ret['datafile']) - + # test that said media file exists assert(media_file_exists(fname)) @@ -657,10 +676,10 @@ def test_permissions(self): # user2 should not be able to delete the tileset created by user1 resp = c2.delete('/api/v1/tilesets/' + ret['uuid'] + "/") assert(resp.status_code == 403) - + # the media file should still exist assert(media_file_exists(fname)) - + # user2 should not be able to rename the tileset created by user1 resp = c2.put('/api/v1/tilesets/' + ret['uuid'] + "/", data='{"name":"newname"}', content_type='application/json') @@ -670,7 +689,7 @@ def test_permissions(self): resp = c1.get("/api/v1/tilesets/") assert(json.loads(resp.content.decode('utf-8'))['count'] == 1) assert(media_file_exists(fname)) - + # user1 should be able to rename or modify their tileset resp = c1.patch('/api/v1/tilesets/' + ret['uuid'] + "/", data='{"name":"newname"}', content_type='application/json') @@ -679,7 +698,7 @@ def test_permissions(self): # apply GET on uuid to ensure that tileset has the newly modified name resp = c1.get("/api/v1/tilesets/" + ret['uuid'] + '/') assert(json.loads(resp.content.decode('utf-8'))['name'] == 'newname') - + # the media file should still exist with the same name assert(media_file_exists(fname))