From 328f1b8129d0c605788b5f51a35b7e26061802a2 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 16 Feb 2023 09:19:59 +0100 Subject: [PATCH 01/35] refactor vector layers, views and layers --- docs/usage.rst | 4 +- test_vectortiles/test_app/functions.py | 5 + test_vectortiles/test_app/views.py | 206 ++++++----------- test_vectortiles/test_app/vt_layers.py | 215 ++++++++++++++++++ vectortiles/__init__.py | 5 + vectortiles/backends/__init__.py | 108 +++++++++ .../postgis/__init__.py} | 11 +- .../{ => backends}/postgis/functions.py | 0 .../mixins.py => backends/python/__init__.py} | 13 +- vectortiles/mapbox/__init__.py | 0 vectortiles/mapbox/views.py | 8 - vectortiles/mixins.py | 176 +++----------- vectortiles/postgis/__init__.py | 0 vectortiles/postgis/views.py | 8 - vectortiles/rest_framework/views.py | 14 ++ vectortiles/settings.py | 2 + vectortiles/tests/test_functions.py | 2 +- vectortiles/tests/test_mixins.py | 6 +- vectortiles/tests/test_views.py | 137 ++--------- vectortiles/views.py | 29 ++- 20 files changed, 509 insertions(+), 440 deletions(-) create mode 100644 test_vectortiles/test_app/functions.py create mode 100644 test_vectortiles/test_app/vt_layers.py create mode 100644 vectortiles/backends/__init__.py rename vectortiles/{postgis/mixins.py => backends/postgis/__init__.py} (86%) rename vectortiles/{ => backends}/postgis/functions.py (100%) rename vectortiles/{mapbox/mixins.py => backends/python/__init__.py} (89%) delete mode 100644 vectortiles/mapbox/__init__.py delete mode 100644 vectortiles/mapbox/views.py delete mode 100644 vectortiles/postgis/__init__.py delete mode 100644 vectortiles/postgis/views.py create mode 100644 vectortiles/rest_framework/views.py diff --git a/docs/usage.rst b/docs/usage.rst index 9be73bc..1fed1b8 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -102,7 +102,7 @@ Django Rest Framework from vectortiles.rest_framework.renderers import MVTRenderer - class FeatureAPIView(PostgisBaseVectorTile, APIView): + class FeatureAPIView(BaseVectorTile, APIView): vector_tile_queryset = Feature.objects.all() vector_tile_layer_name = "features" vector_tile_fields = ('name', ) @@ -122,7 +122,7 @@ Django Rest Framework # or extending viewset - class FeatureViewSet(PostgisBaseVectorTile, viewsets.ModelViewSet): + class FeatureViewSet(BaseVectorTile, viewsets.ModelViewSet): queryset = Feature.objects.all() vector_tile_layer_name = "features" vector_tile_fields = ('name', ) diff --git a/test_vectortiles/test_app/functions.py b/test_vectortiles/test_app/functions.py new file mode 100644 index 0000000..65dd251 --- /dev/null +++ b/test_vectortiles/test_app/functions.py @@ -0,0 +1,5 @@ +from django.contrib.gis.db.models.functions import GeomOutputGeoFunc + + +class SimplifyPreserveTopology(GeomOutputGeoFunc): + pass diff --git a/test_vectortiles/test_app/views.py b/test_vectortiles/test_app/views.py index 13cac99..f9f1326 100644 --- a/test_vectortiles/test_app/views.py +++ b/test_vectortiles/test_app/views.py @@ -1,117 +1,92 @@ +from hashlib import md5 + +from django.core.cache import cache from django.urls import reverse -from django.views.generic import DetailView, ListView, TemplateView +from django.views.generic import DetailView, TemplateView from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.views import APIView -from test_vectortiles.test_app.models import Feature, Layer -from vectortiles.mapbox.views import MVTView as MapboxMVTView -from vectortiles.mixins import BaseVectorTileView, VectorLayer -from vectortiles.postgis.mixins import PostgisBaseVectorTile -from vectortiles.postgis.views import MVTView as PostgisMVTView +from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer +from test_vectortiles.test_app.vt_layers import ( + FeatureLayerVectorLayer, + FeatureVectorLayer, + FeatureLayerFilteredByDateVectorLayer, CityCentroidVectorLayer, FullDataFeatureVectorLayer, RegionVectorLayer, + CommuneVectorLayer, DepartementVectorLayer, EPCIVectorLayer, SurfaceHydrographiqueVectorLayer, + VoieFerreeVectorLayer, TronconRouteVectorLayer, BatimentVectorLayer, TerrainDeSportVectorLayer, +) +from vectortiles.mixins import BaseVectorTileView from vectortiles.rest_framework.renderers import MVTRenderer +from vectortiles.rest_framework.views import MVTAPIView # test at feature level and at layer level -from vectortiles.views import TileJSONView - - -class MapboxFeatureView(MapboxMVTView, ListView): - model = Feature - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) - vector_tile_queryset_limit = 100 +from vectortiles.views import MVTView, TileJSONView -class MapboxLayerView(MapboxMVTView, DetailView): - model = Layer - vector_tile_fields = ("name",) +class FeatureVectorLayers: + layers = [FeatureVectorLayer()] - def get_vector_tile_layer_name(self): - return self.get_object().name - def get_vector_tile_queryset(self): - return self.get_object().features.all() +class FeatureView(FeatureVectorLayers, MVTView): + """Simple model View""" - def get(self, request, *args, **kwargs): - self.object = self.get_object() - return BaseVectorTileView.get( - self, - request=request, - z=kwargs.get("z"), - x=kwargs.get("x"), - y=kwargs.get("y"), - ) +class FeatureTileJSONView(FeatureVectorLayers, TileJSONView): + """Simple model TileJSON View""" -class PostGISFeatureView(PostgisMVTView, ListView): - model = Feature - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) - vector_tile_queryset_limit = 100 + vector_tile_tilejson_name = "My feature dataset" + vector_tile_tilejson_attribution = "@IGN - BD Topo 12/2022" + vector_tile_tilejson_description = "My dataset" -class PostGISFeatureViewWithManualVectorTileQuerySet(PostgisMVTView, DetailView): - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) +class MultipleVectorLayers: + def get_layers(self): + return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.filter(include_in_tilejson=True)] + #return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.all()] + return [RegionVectorLayer(), DepartementVectorLayer(), EPCIVectorLayer(), CommuneVectorLayer(), + SurfaceHydrographiqueVectorLayer(), BatimentVectorLayer(), TronconRouteVectorLayer(), + VoieFerreeVectorLayer(), TerrainDeSportVectorLayer()] - def get(self, request, *args, **kwargs): - self.vector_tile_queryset = Feature.objects.all() - return BaseVectorTileView.get( - self, - request=request, - z=kwargs.get("z"), - x=kwargs.get("x"), - y=kwargs.get("y"), - ) +class LayerView(MultipleVectorLayers, MVTView): + """Multiple tiles in same time, each Layer instance is a tile layer""" + def get_layers_last_update(self): + last_updated_layer = FullDataLayer.objects.all().order_by("-update_datetime").only('update_datetime').first() + return last_updated_layer.update_datetime if last_updated_layer else None -class PostGISFeatureWithDateView(PostgisMVTView, ListView): - queryset = Feature.objects.filter(date="2020-07-07") - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) + def get_content_status(self, z, x, y): + cache_key = md5(f"tilejson-{self.get_layers_last_update()}-{z}-{x}-{y}".encode()).hexdigest() + if cache.has_key(cache_key): + tile, status = cache.get(cache_key), 200 + else: + tile, status = super().get_content_status(z, x, y) + cache.set(cache_key, tile, timeout=3600 * 24 * 30) + return tile, status -class PostGISLayerView(PostgisMVTView, DetailView): - model = Layer - vector_tile_fields = ("name",) - def get_vector_tile_layer_name(self): - return self.get_object().name +class LayerTileJSONView(MultipleVectorLayers, TileJSONView): + """Simple model TileJSON View""" + vector_tile_tilejson_name = "My layers dataset" + vector_tile_tilejson_attribution = "@IGN - BD Topo 12/2022" + vector_tile_tilejson_description = "My dataset" - def get_vector_tile_queryset(self): - return self.get_object().features.all() + def get_tile_url(self): + return reverse("layer-pattern") - def get(self, request, *args, **kwargs): - self.object = self.get_object() - return BaseVectorTileView.get( - self, - request=request, - z=kwargs.get("z"), - x=kwargs.get("x"), - y=kwargs.get("y"), - ) +class FeatureWithDateView(MVTView): + layers = [FeatureLayerFilteredByDateVectorLayer()] -class PostGISDRFFeatureView(PostgisBaseVectorTile, APIView): - vector_tile_queryset = Feature.objects.all() - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) - vector_tile_queryset_limit = 100 - renderer_classes = (MVTRenderer,) - def get(self, request, *args, **kwargs): - return Response( - self.get_tile(kwargs.get("x"), kwargs.get("y"), kwargs.get("z")) - ) +class FeatureAPIView(FeatureVectorLayers, MVTAPIView): + pass -class PostGISDRFFeatureViewSet(PostgisBaseVectorTile, viewsets.ModelViewSet): +class FeatureViewSet(BaseVectorTileView, viewsets.ModelViewSet): queryset = Feature.objects.all() - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) - vector_tile_queryset_limit = 100 + layers = [FeatureVectorLayers()] @action( detail=False, @@ -120,72 +95,19 @@ class PostGISDRFFeatureViewSet(PostgisBaseVectorTile, viewsets.ModelViewSet): url_path=r"tiles/(?P\d+)/(?P\d+)/(?P\d+)", url_name="tile", ) - def tile(self, request, *args, **kwargs): - return Response( - self.get_tile( - x=int(kwargs.get("x")), y=int(kwargs.get("y")), z=int(kwargs.get("z")) - ) - ) + def tile(self, request, z, x, y, *args, **kwargs): + z, x, y = int(z), int(x), int(y) + content, status = self.get_content_status(z, x, y) + return Response(content, status=status) -class MapboxTileJSONFeatureView(TileJSONView): +class TileJSONFeatureView(TileJSONView): vector_tile_tilejson_name = "Feature tileset" - vector_tile_tilejson_description = "generated from mapbox library" - vector_tile_tilejson_attribution = "© JEC" - - def get_tile_url(self): - return reverse("feature-mapbox-pattern") - - def get_vector_layers(self): - return [VectorLayer("features", description="Feature layer").get_vector_layer()] - - -class MapboxTileJSONLayerView(TileJSONView, DetailView): - vector_tile_tilejson_name = "Layer's features tileset" - vector_tile_tilejson_description = "generated from mapbox library" - vector_tile_tilejson_attribution = "© JEC" - model = Layer - - def get_tile_url(self): - return reverse("layer-mapbox-pattern", args=(self.kwargs.get("pk"),)) - - def get_vector_layers(self): - return [ - VectorLayer( - self.get_object().name, - description="Feature layer", - ).get_vector_layer() - ] - - -class PostGISTileJSONFeatureView(TileJSONView): - vector_tile_tilejson_name = "Feature tileset" - vector_tile_tilejson_description = "generated from postgis database" - vector_tile_tilejson_attribution = "© JEC" - - def get_tile_url(self): - return reverse("feature-postgis-pattern") - - def get_vector_layers(self): - return [VectorLayer("features", description="Feature layer").get_vector_layer()] - - -class PostGISTileJSONLayerView(TileJSONView, DetailView): - model = Layer - vector_tile_tilejson_name = "Layer's features tileset" - vector_tile_tilejson_description = "generated from postgis database" + vector_tile_tilejson_description = "feature tileset" vector_tile_tilejson_attribution = "© JEC" def get_tile_url(self): - return reverse("layer-postgis-pattern", args=(self.kwargs.get("pk"),)) - - def get_vector_layers(self): - return [ - VectorLayer( - self.get_object().name, - description="Feature layer", - ).get_vector_layer() - ] + return reverse("feature-pattern") class IndexView(TemplateView): diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py new file mode 100644 index 0000000..9349eed --- /dev/null +++ b/test_vectortiles/test_app/vt_layers.py @@ -0,0 +1,215 @@ +from hashlib import md5 + +from django.contrib.gis.db.models.functions import Centroid, Transform +from django.core.cache import cache +from django.db.models import Q +from django.utils.text import slugify + +from test_vectortiles.test_app.functions import SimplifyPreserveTopology +from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer +from vectortiles import VectorLayer + + +class FeatureVectorLayer(VectorLayer): + model = Feature + vector_tile_layer_name = "features" + vector_tile_fields = ("name",) + vector_tile_queryset_limit = 100 + + +class FeatureLayerVectorLayer(VectorLayer): + model = Layer + vector_tile_fields = ("name",) + + def __init__(self, instance): + self.instance = instance + + def get_tile(self, x, y, z): + cache_key = md5(f"{self.get_vector_tile_layer_id()}-{z}-{x}-{y}".encode()).hexdigest() + if cache.has_key(cache_key): + return cache.get(cache_key) + + else: + tile = super().get_tile(x, y, z) + cache.set(cache_key, tile) + return tile + + def get_vector_tile_layer_id(self): + return slugify(self.instance.name) + + def get_vector_tile_layer_name(self): + return slugify(self.instance.name) + + def get_vector_tile_queryset(self, z, x, y): + return self.instance.features.all() + + def get_vector_tile_layer_max_zoom(self): + return self.instance.max_zoom + + def get_vector_tile_layer_min_zoom(self): + return self.instance.min_zoom + + def get_vector_tile_layer_description(self): + return self.instance.description + + +class CityCentroidVectorLayer(FeatureLayerVectorLayer): + vector_tile_geom_name = "centroid" + + def __init__(self): + self.instance = Layer.objects.get(name="Cities") + + def get_vector_tile_layer_min_zoom(self): + return 6 + + def get_vector_tile_layer_id(self): + return "city-centroid" + + def get_vector_tile_layer_name(self): + return "city-centroid" + + def get_vector_tile_queryset(self, *args, **kwargs): + return self.instance.features.all().annotate(centroid=Centroid("geom")) + + +class FeatureLayerFilteredByDateVectorLayer(VectorLayer): + vector_tile_layer_name = "features" + vector_tile_fields = ("name",) + + def get_vector_tile_queryset(self, *args, **kwargs): + return Feature.objects.filter(date="2020-07-07") + + +class FullDataFeatureVectorLayer(VectorLayer): + vector_tile_fields = ("properties",) + + def __init__(self, instance): + self.instance = instance + + def get_tile(self, x, y, z): + cache_key = md5(f"{self.get_vector_tile_layer_id()}-{self.instance.update_datetime}-{z}-{x}-{y}".encode()).hexdigest() + if cache.has_key(cache_key): + return cache.get(cache_key) + + else: + tile = super().get_tile(x, y, z) + cache.set(cache_key, tile, timeout=3600 * 24 * 30) + return tile + + def get_vector_tile_layer_id(self): + return slugify(self.instance.name) + + def get_vector_tile_layer_name(self): + return slugify(self.instance.name) + + def get_vector_tile_queryset(self, z, x, y): + qs = self.instance.features.all() + if self.instance.name == "troncon_de_route": + if z in range(self.get_vector_tile_layer_min_zoom(), 9): + qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) + elif z in range(9, 12): + qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", ])) + elif z in range(12, 15): + qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", + 'Route à 1 chaussée', ])) + elif self.instance.name == "zone_de_vegetation": + if z < 15: + qs = qs.exclude(properties__nature__in=["Verger", 'Vigne', 'Haie', 'Lande ligneuse', 'Peupleraie', 'Bois']) + return qs + + def get_vector_tile_layer_max_zoom(self): + return self.instance.max_zoom + + def get_vector_tile_layer_min_zoom(self): + return self.instance.min_zoom + + def get_vector_tile_layer_description(self): + return self.instance.description + + +class FullDataLayerOptimizeVectorLayer(FullDataFeatureVectorLayer): + vector_tile_geom_name = "simplified_geom" + + def get_vector_tile_queryset(self, *args, **kwargs): + qs = super().get_vector_tile_queryset(*args, **kwargs) + z = args[0] + simplifications = { + 0: 156543, + 1: 78272, + 2: 39136, + 3: 19568, + 4: 9784, + 5: 4892, + 6: 2446, + 7: 1223, + 8: 611.496, + 9: 305.748, + 10: 152.874, + 11: 76.437, + 12: 38.219, + 13: 19.109, + 14: 9.555, + 15: 4.777, + 16: 2.389, + 17: 1.194, + 18: 0.597, + 19: 0.299, + 20: 0.149, + } + return qs.annotate(simplified_geom=SimplifyPreserveTopology(Transform("geom", 3857), simplifications.get(z))) + + +class RegionVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="region") + + +class DepartementVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="departement") + + +class EPCIVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="epci") + + +class CommuneVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="commune") + + +class SurfaceHydrographiqueVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="surface_hydrographique") + + +class VoieFerreeVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="troncon_de_voie_ferree") + + +class TronconRouteVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="troncon_de_route") + + def get_vector_tile_queryset(self, z, x, y): + qs = super().get_vector_tile_queryset(z, x, y) + + if z in range(self.get_vector_tile_layer_min_zoom(), 9): + qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) + elif z in range(9, 12): + qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", ])) + elif z in range(12, 15): + qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", 'Route à 1 chaussée',])) + return qs + + +class BatimentVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="batiment") + + +class TerrainDeSportVectorLayer(FullDataFeatureVectorLayer): + def __init__(self): + self.instance = FullDataLayer.objects.get(name="terrain_de_sport") diff --git a/vectortiles/__init__.py b/vectortiles/__init__.py index e69de29..cd9bdde 100644 --- a/vectortiles/__init__.py +++ b/vectortiles/__init__.py @@ -0,0 +1,5 @@ +from django.utils.module_loading import import_string + +from vectortiles import settings as app_settings + +VectorLayer = import_string(f"{app_settings.VECTOR_TILES_BACKEND}.VectorLayer") diff --git a/vectortiles/backends/__init__.py b/vectortiles/backends/__init__.py new file mode 100644 index 0000000..96a03a5 --- /dev/null +++ b/vectortiles/backends/__init__.py @@ -0,0 +1,108 @@ +import mercantile + + +class BaseVectorLayerMixin: + """ + Base Mixin to handle vector tile generation + """ + + model = None + queryset = None + + vector_tile_queryset = None + vector_tile_queryset_limit = None # if you want to limit element in tile + vector_tile_layer_name = None # name for data layer in vector tile + vector_tile_geom_name = "geom" # geom field to consider in qs + vector_tile_fields = None # other fields to include from qs + vector_tile_generation = ( + None # use mapbox if you installed [mapbox] sub-dependencies + ) + vector_tile_extent = 4096 # define tile extent + vector_tile_buffer = ( + 256 # define buffer around tiles (intersected polygon display without borders) + ) + vector_tile_clip_geom = ( + True # define if feature geometries should be clipped in tile + ) + vector_tile_layer_id = "" + vector_tile_layer_description = "" + vector_tile_layer_min_zoom = 0 + vector_tile_layer_max_zoom = 22 + + def check_in_zoom_levels(self, z): + return self.get_vector_tile_layer_min_zoom() <= z <= self.get_vector_tile_layer_max_zoom() + + def get_vector_tile_layer_id(self): + return self.vector_tile_layer_id + + def get_vector_tile_layer_description(self): + return self.vector_tile_layer_description + + def get_vector_tile_layer_min_zoom(self): + return self.vector_tile_layer_min_zoom + + def get_vector_tile_layer_max_zoom(self): + return self.vector_tile_layer_max_zoom + + def get_tilejson_vector_layer(self): + return { + "id": self.get_vector_tile_layer_id(), + "description": self.get_vector_tile_layer_description(), + "fields": {}, # self.layer_fields(layer), + "minzoom": self.get_vector_tile_layer_min_zoom(), + "maxzoom": self.get_vector_tile_layer_max_zoom(), + } + + @classmethod + def get_bounds(cls, x, y, z): + """ + Get extent from xyz tile extent to 3857 + + :param x: longitude coordinate tile + :type x: int + :param y: latitude coordinate tile + :type y: int + :param z: zoom level + :type z: int + + :return: xmin, ymin, xmax, ymax in 3857 coordinate system + :rtype: tuple + """ + return mercantile.xy_bounds(x, y, z) + + def get_queryset(self): + if self.queryset is not None: + return self.queryset + return self.model.objects.all() + + def get_vector_tile_queryset(self, *args, **kwargs): + """Get feature queryset in tile dynamically""" + return ( + self.vector_tile_queryset + if self.vector_tile_queryset is not None + else self.get_queryset() + ) + + def get_vector_tile_queryset_limit(self): + """Get feature limit by tile dynamically""" + return self.vector_tile_queryset_limit + + def get_vector_tile_layer_name(self): + """Get layer name in tile dynamically""" + return self.vector_tile_layer_name + + def get_tile(self, x, y, z): + """ + Generate a mapbox vector tile as bytearray + + :param x: longitude coordinate tile + :type x: int + :param y: latitude coordinate tile + :type y: int + :param z: zoom level + :type z: int + + :return: Mapbox Vector Tile + :rtype: bytearray + """ + raise NotImplementedError() diff --git a/vectortiles/postgis/mixins.py b/vectortiles/backends/postgis/__init__.py similarity index 86% rename from vectortiles/postgis/mixins.py rename to vectortiles/backends/postgis/__init__.py index 88bdfec..57bcbae 100644 --- a/vectortiles/postgis/mixins.py +++ b/vectortiles/backends/postgis/__init__.py @@ -1,16 +1,17 @@ from django.contrib.gis.db.models.functions import Transform from django.db import connection -from vectortiles.mixins import BaseVectorTileMixin -from vectortiles.postgis.functions import AsMVTGeom, MakeEnvelope +from vectortiles.backends import BaseVectorLayerMixin +from vectortiles.backends.postgis.functions import MakeEnvelope, AsMVTGeom -class PostgisBaseVectorTile(BaseVectorTileMixin): +class VectorLayer(BaseVectorLayerMixin): def get_tile(self, x, y, z): + if not self.check_in_zoom_levels(z): + return b"" + features = self.get_vector_tile_queryset(z, x, y) # get tile coordinates from x, y and z xmin, ymin, xmax, ymax = self.get_bounds(x, y, z) - features = self.get_vector_tile_queryset() - # keep features intersecting tile filters = { # GeoFuncMixin implicitly transforms to SRID of geom diff --git a/vectortiles/postgis/functions.py b/vectortiles/backends/postgis/functions.py similarity index 100% rename from vectortiles/postgis/functions.py rename to vectortiles/backends/postgis/functions.py diff --git a/vectortiles/mapbox/mixins.py b/vectortiles/backends/python/__init__.py similarity index 89% rename from vectortiles/mapbox/mixins.py rename to vectortiles/backends/python/__init__.py index d52fd03..fd51711 100644 --- a/vectortiles/mapbox/mixins.py +++ b/vectortiles/backends/python/__init__.py @@ -5,22 +5,23 @@ from django.contrib.gis.geos import Polygon from django.db.models import F -from vectortiles.mixins import BaseVectorTileMixin +from vectortiles.backends import BaseVectorLayerMixin -class MapboxBaseVectorTile(BaseVectorTileMixin): - vector_tile_generation = "mapbox" - +class VectorLayer(BaseVectorLayerMixin): def pixel_length(self, zoom, size): radius = 6378137 circum = 2 * math.pi * radius return circum / size / 2 ** int(zoom) def get_tile(self, x, y, z): + if not self.check_in_zoom_levels(z): + return b"" + + features = self.get_vector_tile_queryset(z, x, y) + # get tile coordinates from x, y and z west, south, east, north = self.get_bounds(x, y, z) - features = self.get_vector_tile_queryset() - bbox = Polygon.from_bbox((west, south, east, north)) bbox.srid = 3857 diff --git a/vectortiles/mapbox/__init__.py b/vectortiles/mapbox/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/vectortiles/mapbox/views.py b/vectortiles/mapbox/views.py deleted file mode 100644 index 9effaac..0000000 --- a/vectortiles/mapbox/views.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.views import View - -from vectortiles.mapbox.mixins import MapboxBaseVectorTile -from vectortiles.mixins import BaseVectorTileView - - -class MVTView(BaseVectorTileView, MapboxBaseVectorTile, View): - pass diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index d06836f..fafc58a 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -1,128 +1,26 @@ -from urllib.parse import unquote - -import mercantile -from django.http import HttpResponse +from urllib.parse import unquote, urljoin from vectortiles import settings as app_settings -class BaseVectorTileMixin: - """ - Base Mixin to handle vector tile generation - """ - - vector_tile_content_type = "application/x-protobuf" - vector_tile_queryset = None - vector_tile_queryset_limit = None - vector_tile_layer_name = None # name for data layer in vector tile - vector_tile_geom_name = "geom" # geom field to consider in qs - vector_tile_fields = None # other fields to include from qs - vector_tile_generation = ( - None # use mapbox if you installed [mapbox] sub-dependencies - ) - vector_tile_extent = 4096 # define tile extent - vector_tile_buffer = ( - 256 # define buffer around tiles (intersected polygon display without borders) - ) - vector_tile_clip_geom = ( - True # define if feature geometries should be clipped in tile - ) - - @classmethod - def get_bounds(cls, x, y, z): - """ - Get extent from xyz tile extent to 3857 - - :param x: longitude coordinate tile - :type x: int - :param y: latitude coordinate tile - :type y: int - :param z: zoom level - :type z: int - - :return: xmin, ymin, xmax, ymax in 3857 coordinate system - :rtype: tuple - """ - return mercantile.xy_bounds(x, y, z) - - def get_vector_tile_queryset(self): - """Get feature queryset in tile dynamically""" - return ( - self.vector_tile_queryset - if self.vector_tile_queryset is not None - else self.get_queryset() - ) - - def get_vector_tile_queryset_limit(self): - """Get feature limit by tile dynamically""" - return self.vector_tile_queryset_limit - - def get_vector_tile_layer_name(self): - """Get layer name in tile dynamically""" - return self.vector_tile_layer_name - - def get_tile(self, x, y, z): - """ - Generate a mapbox vector tile as bytearray - - :param x: longitude coordinate tile - :type x: int - :param y: latitude coordinate tile - :type y: int - :param z: zoom level - :type z: int - - :return: Mapbox Vector Tile - :rtype: bytearray - """ - raise NotImplementedError() - - -class VectorLayer: - vector_tile_layer_id = "" - vector_tile_layer_description = "" - vector_tile_layer_min_zoom = 0 - vector_tile_layer_max_zoom = 22 - - def __init__(self, id_layer, description="", min_zoom=0, max_zoom=22): - self.vector_tile_layer_id = id_layer - self.vector_tile_layer_description = description - self.vector_tile_layer_min_zoom = min_zoom - self.vector_tile_layer_max_zoom = max_zoom - - def get_vector_tile_layer_id(self): - return self.vector_tile_layer_id - - def get_vector_tile_layer_description(self): - return self.vector_tile_layer_description - - def get_vector_tile_layer_min_zoom(self): - return self.vector_tile_layer_min_zoom - - def get_vector_tile_layer_max_zoom(self): - return self.vector_tile_layer_max_zoom - - def get_vector_layer(self): - return { - "id": self.get_vector_tile_layer_id(), - "description": self.get_vector_tile_layer_description(), - "fields": {}, # self.layer_fields(layer), - "minzoom": self.get_vector_tile_layer_min_zoom(), - "maxzoom": self.get_vector_tile_layer_max_zoom(), - } +class BaseVectorView: + layers = None + def get_layers(self): + return self.layers or [] -class BaseTileJSONMixin: + +class BaseTileJSONView(BaseVectorView): vector_tile_tilejson_name = "" vector_tile_tilejson_attribution = "" vector_tile_tilejson_description = "" def get_vector_tile_tilejson_min_zoom(self): - min_zoom = min(item["minzoom"] for item in self.get_vector_layers()) + min_zoom = min(layer.get_tilejson_vector_layer()["minzoom"] for layer in self.get_layers()) return min_zoom or 0 def get_vector_tile_tilejson_max_zoom(self): - max_zoom = max(item["maxzoom"] for item in self.get_vector_layers()) + max_zoom = max(layer.get_tilejson_vector_layer()["maxzoom"] for layer in self.get_layers()) return max_zoom or 22 def get_vector_tile_tilejson_attribution(self): @@ -134,19 +32,14 @@ def get_vector_tile_tilejson_description(self): def get_vector_tile_tilejson_name(self): return self.vector_tile_tilejson_name - def get_tile_urls(self, tile_url, base_url=""): - # if app_settings.TERRA_TILES_HOSTNAMES: - # return [ - # unquote(urljoin(hostname, tile_url)) - # for hostname in app_settings.VECTOR_TILES_HOSTNAMES - # ] - # else: - return [unquote(tile_url)] - - def get_vector_layers(self): - raise NotImplementedError( - """ you should implement get_vector_layers to return a VectorLayer list """ - ) + def get_tile_urls(self, tile_url): + if app_settings.VECTOR_TILES_HOSTNAMES: + return [ + unquote(urljoin(hostname, tile_url)) + for hostname in app_settings.VECTOR_TILES_HOSTNAMES + ] + else: + return [unquote(self.request.build_absolute_uri(tile_url))] def get_tilejson(self, tile_url, version="3.0.0"): # https://github.com/mapbox/tilejson-spec/tree/3.0/3.0.0 @@ -160,30 +53,23 @@ def get_tilejson(self, tile_url, version="3.0.0"): # center "attribution": self.get_vector_tile_tilejson_attribution(), "description": self.get_vector_tile_tilejson_description(), - "vector_layers": self.get_vector_layers(), + "vector_layers": [ + layer.get_tilejson_vector_layer() for layer in self.get_layers() + ], } -class BaseVectorTileView: - """Base mixin to handle vector tile in a djang oView""" +class BaseVectorTileView(BaseVectorView): + """Base mixin to handle vector tile in a django View""" content_type = app_settings.VECTOR_TILES_CONTENT_TYPE - def get(self, request, z, x, y): - """ - Handle GET request to serve tile - - :param request: - :type request: HttpRequest - :param x: longitude coordinate tile - :type x: int - :param y: latitude coordinate tile - :type y: int - :param z: zoom level - :type z: int - - :rtype HTTPResponse - """ - content = self.get_tile(x, y, z) - status = 200 if content else 204 - return HttpResponse(content, content_type=self.content_type, status=status) + def get_layer_tiles(self, z, x, y): + layers = self.get_layers() + if layers: + return b"".join(layer.get_tile(x, y, z) for layer in layers) + raise Exception("No layers defined") + + def get_content_status(self, z, x, y): + content = self.get_layer_tiles(z, x, y) + return (content, 200) if content else (content, 204) diff --git a/vectortiles/postgis/__init__.py b/vectortiles/postgis/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/vectortiles/postgis/views.py b/vectortiles/postgis/views.py deleted file mode 100644 index 56b0529..0000000 --- a/vectortiles/postgis/views.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.views import View - -from vectortiles.mixins import BaseVectorTileView -from vectortiles.postgis.mixins import PostgisBaseVectorTile - - -class MVTView(BaseVectorTileView, PostgisBaseVectorTile, View): - pass diff --git a/vectortiles/rest_framework/views.py b/vectortiles/rest_framework/views.py new file mode 100644 index 0000000..4de3665 --- /dev/null +++ b/vectortiles/rest_framework/views.py @@ -0,0 +1,14 @@ +from rest_framework.response import Response +from rest_framework.views import APIView + +from vectortiles.mixins import BaseVectorTileView +from vectortiles.rest_framework.renderers import MVTRenderer + + +class MVTAPIView(BaseVectorTileView, APIView): + renderer_classes = (MVTRenderer,) + + def get(self, request, z, x, y, *args, **kwargs): + z, x, y = int(z), int(x), int(y) + content, status = self.get_content_status(z, x, y) + return Response(content, status=status) diff --git a/vectortiles/settings.py b/vectortiles/settings.py index c7b20da..a6962c9 100644 --- a/vectortiles/settings.py +++ b/vectortiles/settings.py @@ -4,3 +4,5 @@ settings, "VECTOR_TILES_CONTENT_TYPE", "application/vnd.mapbox-vector-tile" ) VECTOR_TILES_EXTENSION = getattr(settings, "VECTOR_TILES_EXTENSION", "mvt") +VECTOR_TILES_BACKEND = "vectortiles.backends.postgis" # to use python backend, set to 'vectortiles.backends.python' +VECTOR_TILES_HOSTNAMES = getattr(settings, "VECTOR_TILES_HOSTNAMES", None) diff --git a/vectortiles/tests/test_functions.py b/vectortiles/tests/test_functions.py index 874f7c8..1b8d41d 100644 --- a/vectortiles/tests/test_functions.py +++ b/vectortiles/tests/test_functions.py @@ -3,7 +3,7 @@ from django.test import TestCase from test_vectortiles.test_app.models import Feature -from vectortiles.postgis.functions import MakeEnvelope +from vectortiles.backends.postgis.functions import MakeEnvelope class MakeEnvelopeTestCase(TestCase): diff --git a/vectortiles/tests/test_mixins.py b/vectortiles/tests/test_mixins.py index e2deee3..db412cf 100644 --- a/vectortiles/tests/test_mixins.py +++ b/vectortiles/tests/test_mixins.py @@ -1,17 +1,17 @@ from django.test import TestCase -from vectortiles.mixins import BaseTileJSONMixin, BaseVectorTileMixin +from vectortiles.mixins import BaseTileJSONView, BaseVectorTileView class BaseVectorTileMixinTestCase(TestCase): def test_raise_not_implemented(self): with self.assertRaises(NotImplementedError): - instance = BaseVectorTileMixin() + instance = BaseVectorTileView() instance.get_tile(0, 0, 0) class BaseTileJSONMixinTestCase(TestCase): def test_raise_not_implemented(self): with self.assertRaises(NotImplementedError): - instance = BaseTileJSONMixin() + instance = BaseTileJSONView() instance.get_vector_layers() diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 901d341..a143bbe 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -26,16 +26,12 @@ def test_num_queries_equals_one(self): self.maxDiff = None with self.assertNumQueries(1): self.client.get( - reverse( - "feature-postgis-with-manual-vector-tile-queryset", args=(0, 0, 0) - ) + reverse("feature-with-manual-vector-tile-queryset", args=(0, 0, 0)) ) - def test_mapbox_layer(self): + def test_layer(self): self.maxDiff = None - response = self.client.get( - reverse("layer-mapbox", args=(self.layer.pk, 0, 0, 0)) - ) + response = self.client.get(reverse("layer", args=(self.layer.pk, 0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -66,43 +62,9 @@ def test_mapbox_layer(self): content, ) - def test_mapbox_features(self): + def test_features(self): self.maxDiff = None - response = self.client.get(reverse("feature-mapbox", args=(0, 0, 0))) - self.assertEqual(response.status_code, 200) - content = mapbox_vector_tile.decode(response.content) - self.assertDictEqual( - content, - { - "features": { - "extent": 4096, - "version": 1, - "features": [ - { - "geometry": {"type": "Point", "coordinates": [2048, 2048]}, - "properties": {"name": "feat1"}, - "id": 0, - "type": 1, - }, - { - "geometry": { - "type": "LineString", - "coordinates": [[2048, 2048], [2059, 2059]], - }, - "properties": {"name": "feat2"}, - "id": 0, - "type": 2, - }, - ], - } - }, - ) - - def test_postgis_layer(self): - self.maxDiff = None - response = self.client.get( - reverse("layer-postgis", args=(self.layer.pk, 0, 0, 0)) - ) + response = self.client.get(reverse("feature", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -132,9 +94,9 @@ def test_postgis_layer(self): }, ) - def test_postgis_features(self): + def test_layer(self): self.maxDiff = None - response = self.client.get(reverse("feature-postgis", args=(0, 0, 0))) + response = self.client.get(reverse("layer", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -164,9 +126,9 @@ def test_postgis_features(self): }, ) - def test_postgis_drf_api_features(self): + def test_drf_api_features(self): self.maxDiff = None - response = self.client.get(reverse("feature-postgis-drf", args=(0, 0, 0))) + response = self.client.get(reverse("feature-drf", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -196,7 +158,7 @@ def test_postgis_drf_api_features(self): }, ) - def test_postgis_drf_viewset_features(self): + def test_drf_viewset_features(self): self.maxDiff = None response = self.client.get(reverse("feature-drf-viewset-tile", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) @@ -228,9 +190,9 @@ def test_postgis_drf_viewset_features(self): }, ) - def test_postgis_features_with_filtered_date(self): + def test_features_with_filtered_date(self): self.maxDiff = None - response = self.client.get(reverse("feature-date-postgis", args=(0, 0, 0))) + response = self.client.get(reverse("feature-date", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -253,23 +215,21 @@ def test_postgis_features_with_filtered_date(self): class VectorTileTileJSONTestCase(VectorTileBaseTest): - def test_mapbox_layer(self): + def test_layer(self): self.maxDiff = None - response = self.client.get( - reverse("layer-mapbox-tilejson", args=(self.layer.pk,)) - ) + response = self.client.get(reverse("layer-tilejson", args=(self.layer.pk,))) self.assertEqual(response.status_code, 200) content = response.json() self.assertDictEqual( content, { "attribution": "© JEC", - "description": "generated from mapbox library", + "description": "generated from data", "maxzoom": 22, "minzoom": 0, "name": "Layer's features tileset", "tilejson": "3.0.0", - "tiles": ["/layer/2/mapbox/tile/{z}/{x}/{y}"], + "tiles": ["/layer/2/tile/{z}/{x}/{y}"], "vector_layers": [ { "description": "Feature layer", @@ -282,21 +242,21 @@ def test_mapbox_layer(self): }, ) - def test_mapbox_features(self): + def test_features(self): self.maxDiff = None - response = self.client.get(reverse("feature-mapbox-tilejson")) + response = self.client.get(reverse("feature-tilejson")) self.assertEqual(response.status_code, 200) content = response.json() self.assertDictEqual( content, { "attribution": "© JEC", - "description": "generated from mapbox library", + "description": "feature tileset", "maxzoom": 22, "minzoom": 0, "name": "Feature tileset", "tilejson": "3.0.0", - "tiles": ["/features/mapbox/tile/{z}/{x}/{y}"], + "tiles": ["/features/tile/{z}/{x}/{y}"], "vector_layers": [ { "description": "Feature layer", @@ -309,63 +269,6 @@ def test_mapbox_features(self): }, ) - def test_postgis_layer(self): - self.maxDiff = None - response = self.client.get( - reverse("layer-postgis-tilejson", args=(self.layer.pk,)) - ) - self.assertEqual(response.status_code, 200) - content = response.json() - self.assertDictEqual( - content, - { - "attribution": "© JEC", - "description": "generated from postgis database", - "maxzoom": 22, - "minzoom": 0, - "name": "Layer's features tileset", - "tilejson": "3.0.0", - "tiles": ["/layer/2/postgis/tile/{z}/{x}/{y}"], - "vector_layers": [ - { - "description": "Feature layer", - "fields": {}, - "id": "features", - "maxzoom": 22, - "minzoom": 0, - } - ], - }, - ) - - def test_postgis_features(self): - self.maxDiff = None - response = self.client.get(reverse("feature-postgis-tilejson")) - self.assertEqual(response.status_code, 200) - content = response.json() - self.assertDictEqual( - content, - { - "attribution": "© JEC", - "description": "generated from postgis database", - "maxzoom": 22, - "minzoom": 0, - "name": "Feature tileset", - "tilejson": "3.0.0", - "tiles": ["/features/postgis/tile/{z}/{x}/{y}"], - "vector_layers": [ - { - "description": "Feature layer", - "fields": {}, - "id": "features", - "maxzoom": 22, - "minzoom": 0, - } - ], - }, - content, - ) - def test_tilejson_view_default(self): class TestView(TileJSONView): tile_url = "test" diff --git a/vectortiles/views.py b/vectortiles/views.py index b9425d2..a531a2e 100644 --- a/vectortiles/views.py +++ b/vectortiles/views.py @@ -1,10 +1,13 @@ -from django.http import JsonResponse +from hashlib import md5 + +from django.core.cache import cache +from django.http import HttpResponse, JsonResponse from django.views import View -from vectortiles.mixins import BaseTileJSONMixin +from vectortiles.mixins import BaseTileJSONView, BaseVectorTileView -class TileJSONView(BaseTileJSONMixin, View): +class TileJSONView(BaseTileJSONView, View): tile_url = None def get_tile_url(self): @@ -12,3 +15,23 @@ def get_tile_url(self): def get(self, request, *args, **kwargs): return JsonResponse(self.get_tilejson(self.get_tile_url())) + + +class MVTView(BaseVectorTileView, View): + def get(self, request, z, x, y, *args, **kwargs): + """ + Handle GET request to serve tile + + :param request: + :type request: HttpRequest + :param x: longitude coordinate tile + :type x: int + :param y: latitude coordinate tile + :type y: int + :param z: zoom level + :type z: int + + :rtype HTTPResponse + """ + content, status = self.get_content_status(int(z), int(x), int(y)) + return HttpResponse(content, content_type=self.content_type, status=status) From 3f4a9be9a16377aaaec5b8377ec65cf98ceb4a1d Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 16 Feb 2023 09:24:04 +0100 Subject: [PATCH 02/35] fix lint --- .gitignore | 1 + setup.cfg | 2 +- vectortiles/backends/__init__.py | 6 +++++- vectortiles/backends/postgis/__init__.py | 2 +- vectortiles/mixins.py | 8 ++++++-- vectortiles/tests/test_views.py | 2 +- vectortiles/views.py | 3 --- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 7df973b..c74c818 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ venv.bak/ .mypy_cache/ .idea/ /docs/_build/ +cache \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 57093fb..076128f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -ignore = E501,W504 +ignore = E501,W503 exclude = test_vectortiles/settings* [isort] diff --git a/vectortiles/backends/__init__.py b/vectortiles/backends/__init__.py index 96a03a5..afc6d6a 100644 --- a/vectortiles/backends/__init__.py +++ b/vectortiles/backends/__init__.py @@ -30,7 +30,11 @@ class BaseVectorLayerMixin: vector_tile_layer_max_zoom = 22 def check_in_zoom_levels(self, z): - return self.get_vector_tile_layer_min_zoom() <= z <= self.get_vector_tile_layer_max_zoom() + return ( + self.get_vector_tile_layer_min_zoom() + <= z + <= self.get_vector_tile_layer_max_zoom() + ) def get_vector_tile_layer_id(self): return self.vector_tile_layer_id diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index 57bcbae..34f6c4c 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -2,7 +2,7 @@ from django.db import connection from vectortiles.backends import BaseVectorLayerMixin -from vectortiles.backends.postgis.functions import MakeEnvelope, AsMVTGeom +from vectortiles.backends.postgis.functions import AsMVTGeom, MakeEnvelope class VectorLayer(BaseVectorLayerMixin): diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index fafc58a..9a4ec6b 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -16,11 +16,15 @@ class BaseTileJSONView(BaseVectorView): vector_tile_tilejson_description = "" def get_vector_tile_tilejson_min_zoom(self): - min_zoom = min(layer.get_tilejson_vector_layer()["minzoom"] for layer in self.get_layers()) + min_zoom = min( + layer.get_tilejson_vector_layer()["minzoom"] for layer in self.get_layers() + ) return min_zoom or 0 def get_vector_tile_tilejson_max_zoom(self): - max_zoom = max(layer.get_tilejson_vector_layer()["maxzoom"] for layer in self.get_layers()) + max_zoom = max( + layer.get_tilejson_vector_layer()["maxzoom"] for layer in self.get_layers() + ) return max_zoom or 22 def get_vector_tile_tilejson_attribution(self): diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index a143bbe..2eb3a12 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -94,7 +94,7 @@ def test_features(self): }, ) - def test_layer(self): + def test_layer_2(self): self.maxDiff = None response = self.client.get(reverse("layer", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) diff --git a/vectortiles/views.py b/vectortiles/views.py index a531a2e..a808c22 100644 --- a/vectortiles/views.py +++ b/vectortiles/views.py @@ -1,6 +1,3 @@ -from hashlib import md5 - -from django.core.cache import cache from django.http import HttpResponse, JsonResponse from django.views import View From d3f522a0701c8979a26ad20adce822dad8cb47fe Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 16 Feb 2023 09:26:38 +0100 Subject: [PATCH 03/35] fix lint --- test_vectortiles/settings.py | 7 + test_vectortiles/test_app/admin.py | 35 +- .../migrations/0006_auto_20230209_0933.py | 38 ++ .../0007_fulldatafeature_fulldatalayer.py | 66 ++++ .../0008_alter_fulldatafeature_properties.py | 21 ++ .../0009_alter_fulldatalayer_options.py | 17 + ..._fulldatafeature_feature_properties_gin.py | 20 ++ .../0011_fulldatalayer_include_in_tilejson.py | 18 + .../0012_fulldatalayer_update_datetime.py | 18 + ...013_alter_fulldatalayer_update_datetime.py | 18 + test_vectortiles/test_app/models.py | 45 ++- .../test_app/templates/index.html | 326 +++++++++++++++++- test_vectortiles/test_app/views.py | 54 ++- test_vectortiles/test_app/vt_layers.py | 74 +++- test_vectortiles/urls.py | 88 ++--- 15 files changed, 737 insertions(+), 108 deletions(-) create mode 100644 test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py create mode 100644 test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py create mode 100644 test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py create mode 100644 test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py create mode 100644 test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py create mode 100644 test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py create mode 100644 test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py create mode 100644 test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py diff --git a/test_vectortiles/settings.py b/test_vectortiles/settings.py index b400c5b..dbdb181 100644 --- a/test_vectortiles/settings.py +++ b/test_vectortiles/settings.py @@ -120,3 +120,10 @@ # https://docs.djangoproject.com/en/2.2/howto/static-files/ STATIC_URL = "/static/" + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "cache", + } +} diff --git a/test_vectortiles/test_app/admin.py b/test_vectortiles/test_app/admin.py index d0d5be9..f05dddb 100644 --- a/test_vectortiles/test_app/admin.py +++ b/test_vectortiles/test_app/admin.py @@ -1,7 +1,12 @@ from django.contrib import admin from django.contrib.gis.admin import OSMGeoAdmin -from test_vectortiles.test_app.models import Feature +from test_vectortiles.test_app.models import ( + Feature, + FullDataFeature, + FullDataLayer, + Layer, +) @admin.register(Feature) @@ -9,3 +14,31 @@ class FeatureAdmin(OSMGeoAdmin): list_display = ("id", "name", "layer", "date") list_filter = ("layer", "date") search_fields = ("name",) + + +@admin.register(Layer) +class LayerAdmin(admin.ModelAdmin): + list_display = ("id", "name", "attribution", "min_zoom", "max_zoom") + search_fields = ("id", "name", "description", "attribution") + + +@admin.register(FullDataLayer) +class FullDataLayerAdmin(admin.ModelAdmin): + list_display = ( + "id", + "name", + "attribution", + "include_in_tilejson", + "min_zoom", + "max_zoom", + ) + search_fields = ("id", "name", "description", "attribution") + + +@admin.register(FullDataFeature) +class FulDataFeatureAdmin(OSMGeoAdmin): + list_display = ( + "id", + "layer", + ) + list_filter = ("layer",) diff --git a/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py b/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py new file mode 100644 index 0000000..73c883b --- /dev/null +++ b/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.17 on 2023-02-09 09:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0005_alter_feature_date"), + ] + + operations = [ + migrations.AddField( + model_name="layer", + name="attribution", + field=models.CharField(blank=True, default="", max_length=250), + ), + migrations.AddField( + model_name="layer", + name="description", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="layer", + name="max_zoom", + field=models.PositiveSmallIntegerField(default=22), + ), + migrations.AddField( + model_name="layer", + name="min_zoom", + field=models.PositiveSmallIntegerField(default=0), + ), + migrations.AlterField( + model_name="layer", + name="name", + field=models.CharField(max_length=250, unique=True), + ), + ] diff --git a/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py b/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py new file mode 100644 index 0000000..74b34ae --- /dev/null +++ b/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py @@ -0,0 +1,66 @@ +# Generated by Django 3.2.17 on 2023-02-10 08:24 + +import django.contrib.gis.db.models.fields +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0006_auto_20230209_0933"), + ] + + operations = [ + migrations.CreateModel( + name="FullDataLayer", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=250, unique=True)), + ( + "attribution", + models.CharField(blank=True, default="", max_length=250), + ), + ("description", models.TextField(blank=True)), + ("min_zoom", models.PositiveSmallIntegerField(default=0)), + ("max_zoom", models.PositiveSmallIntegerField(default=22)), + ], + ), + migrations.CreateModel( + name="FullDataFeature", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("geom", django.contrib.gis.db.models.fields.GeometryField(srid=4326)), + ("properties", models.JSONField(default=dict)), + ( + "layer", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="features", + to="test_app.fulldatalayer", + ), + ), + ], + options={ + "ordering": ("id",), + }, + ), + ] diff --git a/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py b/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py new file mode 100644 index 0000000..9a9e9e5 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.17 on 2023-02-10 08:29 + +import django.core.serializers.json +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0007_fulldatafeature_fulldatalayer"), + ] + + operations = [ + migrations.AlterField( + model_name="fulldatafeature", + name="properties", + field=models.JSONField( + default=dict, encoder=django.core.serializers.json.DjangoJSONEncoder + ), + ), + ] diff --git a/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py b/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py new file mode 100644 index 0000000..fd22ea6 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.17 on 2023-02-10 14:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0008_alter_fulldatafeature_properties"), + ] + + operations = [ + migrations.AlterModelOptions( + name="fulldatalayer", + options={"ordering": ("name",)}, + ), + ] diff --git a/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py b/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py new file mode 100644 index 0000000..c103a7a --- /dev/null +++ b/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.17 on 2023-02-13 13:44 + +import django.contrib.postgres.indexes +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0009_alter_fulldatalayer_options"), + ] + + operations = [ + migrations.AddIndex( + model_name="fulldatafeature", + index=django.contrib.postgres.indexes.GinIndex( + fields=["properties"], name="feature_properties_gin" + ), + ), + ] diff --git a/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py b/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py new file mode 100644 index 0000000..383e842 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-02-14 16:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0010_fulldatafeature_feature_properties_gin"), + ] + + operations = [ + migrations.AddField( + model_name="fulldatalayer", + name="include_in_tilejson", + field=models.BooleanField(default=False), + ), + ] diff --git a/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py b/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py new file mode 100644 index 0000000..c1fb495 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-02-15 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0011_fulldatalayer_include_in_tilejson"), + ] + + operations = [ + migrations.AddField( + model_name="fulldatalayer", + name="update_datetime", + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py b/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py new file mode 100644 index 0000000..b6cb16b --- /dev/null +++ b/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-02-15 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0012_fulldatalayer_update_datetime"), + ] + + operations = [ + migrations.AlterField( + model_name="fulldatalayer", + name="update_datetime", + field=models.DateTimeField(auto_now=True, db_index=True), + ), + ] diff --git a/test_vectortiles/test_app/models.py b/test_vectortiles/test_app/models.py index 284169b..1ec8bce 100644 --- a/test_vectortiles/test_app/models.py +++ b/test_vectortiles/test_app/models.py @@ -1,8 +1,17 @@ from django.contrib.gis.db import models +from django.contrib.postgres.indexes import GinIndex +from django.core.serializers.json import DjangoJSONEncoder class Layer(models.Model): - name = models.CharField(max_length=250) + name = models.CharField(max_length=250, unique=True) + attribution = models.CharField(max_length=250, default="", blank=True) + description = models.TextField(blank=True) + min_zoom = models.PositiveSmallIntegerField(default=0) + max_zoom = models.PositiveSmallIntegerField(default=22) + + def __str__(self): + return self.name class Feature(models.Model): @@ -15,3 +24,37 @@ class Feature(models.Model): class Meta: ordering = ("id",) + + +class FullDataLayer(models.Model): + name = models.CharField(max_length=250, unique=True) + attribution = models.CharField(max_length=250, default="", blank=True) + description = models.TextField(blank=True) + include_in_tilejson = models.BooleanField(default=False) + min_zoom = models.PositiveSmallIntegerField(default=0) + max_zoom = models.PositiveSmallIntegerField(default=22) + update_datetime = models.DateTimeField(auto_now=True, db_index=True) + + def __str__(self): + return self.name + + class Meta: + ordering = ("name",) + + +class FullDataFeature(models.Model): + geom = models.GeometryField(srid=4326) + layer = models.ForeignKey( + FullDataLayer, + on_delete=models.CASCADE, + related_name="features", + null=True, + blank=True, + ) + properties = models.JSONField(default=dict, encoder=DjangoJSONEncoder) + + class Meta: + ordering = ("id",) + indexes = [ + GinIndex(fields=["properties"], name="feature_properties_gin"), + ] diff --git a/test_vectortiles/test_app/templates/index.html b/test_vectortiles/test_app/templates/index.html index ebabeb2..67dd260 100644 --- a/test_vectortiles/test_app/templates/index.html +++ b/test_vectortiles/test_app/templates/index.html @@ -6,6 +6,12 @@ content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0"> MapBox / MapLibre example + @@ -14,32 +20,326 @@ diff --git a/test_vectortiles/test_app/views.py b/test_vectortiles/test_app/views.py index f9f1326..ac68767 100644 --- a/test_vectortiles/test_app/views.py +++ b/test_vectortiles/test_app/views.py @@ -2,18 +2,25 @@ from django.core.cache import cache from django.urls import reverse -from django.views.generic import DetailView, TemplateView +from django.views.generic import TemplateView from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response -from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer +from test_vectortiles.test_app.models import Feature, FullDataLayer from test_vectortiles.test_app.vt_layers import ( - FeatureLayerVectorLayer, + BatimentVectorLayer, + CommuneVectorLayer, + DepartementVectorLayer, + EPCIVectorLayer, + FeatureLayerFilteredByDateVectorLayer, FeatureVectorLayer, - FeatureLayerFilteredByDateVectorLayer, CityCentroidVectorLayer, FullDataFeatureVectorLayer, RegionVectorLayer, - CommuneVectorLayer, DepartementVectorLayer, EPCIVectorLayer, SurfaceHydrographiqueVectorLayer, - VoieFerreeVectorLayer, TronconRouteVectorLayer, BatimentVectorLayer, TerrainDeSportVectorLayer, + FullDataFeatureVectorLayer, + RegionVectorLayer, + SurfaceHydrographiqueVectorLayer, + TerrainDeSportVectorLayer, + TronconRouteVectorLayer, + VoieFerreeVectorLayer, ) from vectortiles.mixins import BaseVectorTileView from vectortiles.rest_framework.renderers import MVTRenderer @@ -41,23 +48,41 @@ class FeatureTileJSONView(FeatureVectorLayers, TileJSONView): class MultipleVectorLayers: def get_layers(self): - return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.filter(include_in_tilejson=True)] - #return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.all()] - return [RegionVectorLayer(), DepartementVectorLayer(), EPCIVectorLayer(), CommuneVectorLayer(), - SurfaceHydrographiqueVectorLayer(), BatimentVectorLayer(), TronconRouteVectorLayer(), - VoieFerreeVectorLayer(), TerrainDeSportVectorLayer()] + return [ + FullDataFeatureVectorLayer(layer) + for layer in FullDataLayer.objects.filter(include_in_tilejson=True) + ] + # return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.all()] + return [ + RegionVectorLayer(), + DepartementVectorLayer(), + EPCIVectorLayer(), + CommuneVectorLayer(), + SurfaceHydrographiqueVectorLayer(), + BatimentVectorLayer(), + TronconRouteVectorLayer(), + VoieFerreeVectorLayer(), + TerrainDeSportVectorLayer(), + ] class LayerView(MultipleVectorLayers, MVTView): """Multiple tiles in same time, each Layer instance is a tile layer""" def get_layers_last_update(self): - last_updated_layer = FullDataLayer.objects.all().order_by("-update_datetime").only('update_datetime').first() + last_updated_layer = ( + FullDataLayer.objects.all() + .order_by("-update_datetime") + .only("update_datetime") + .first() + ) return last_updated_layer.update_datetime if last_updated_layer else None def get_content_status(self, z, x, y): - cache_key = md5(f"tilejson-{self.get_layers_last_update()}-{z}-{x}-{y}".encode()).hexdigest() - if cache.has_key(cache_key): + cache_key = md5( + f"tilejson-{self.get_layers_last_update()}-{z}-{x}-{y}".encode() + ).hexdigest() + if cache.has_key(cache_key): # NOQA W601 tile, status = cache.get(cache_key), 200 else: @@ -68,6 +93,7 @@ def get_content_status(self, z, x, y): class LayerTileJSONView(MultipleVectorLayers, TileJSONView): """Simple model TileJSON View""" + vector_tile_tilejson_name = "My layers dataset" vector_tile_tilejson_attribution = "@IGN - BD Topo 12/2022" vector_tile_tilejson_description = "My dataset" diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py index 9349eed..da69853 100644 --- a/test_vectortiles/test_app/vt_layers.py +++ b/test_vectortiles/test_app/vt_layers.py @@ -6,7 +6,7 @@ from django.utils.text import slugify from test_vectortiles.test_app.functions import SimplifyPreserveTopology -from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer +from test_vectortiles.test_app.models import Feature, FullDataLayer, Layer from vectortiles import VectorLayer @@ -25,8 +25,10 @@ def __init__(self, instance): self.instance = instance def get_tile(self, x, y, z): - cache_key = md5(f"{self.get_vector_tile_layer_id()}-{z}-{x}-{y}".encode()).hexdigest() - if cache.has_key(cache_key): + cache_key = md5( + f"{self.get_vector_tile_layer_id()}-{z}-{x}-{y}".encode() + ).hexdigest() + if cache.has_key(cache_key): # NOQA W601 return cache.get(cache_key) else: @@ -87,8 +89,10 @@ def __init__(self, instance): self.instance = instance def get_tile(self, x, y, z): - cache_key = md5(f"{self.get_vector_tile_layer_id()}-{self.instance.update_datetime}-{z}-{x}-{y}".encode()).hexdigest() - if cache.has_key(cache_key): + cache_key = md5( + f"{self.get_vector_tile_layer_id()}-{self.instance.update_datetime}-{z}-{x}-{y}".encode() + ).hexdigest() + if cache.has_key(cache_key): # NOQA W601 return cache.get(cache_key) else: @@ -108,13 +112,38 @@ def get_vector_tile_queryset(self, z, x, y): if z in range(self.get_vector_tile_layer_min_zoom(), 9): qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) elif z in range(9, 12): - qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", ])) + qs = qs.filter( + Q( + properties__nature__in=[ + "Type autoroutier", + "Route à 2 chaussées", + "Bretelle", + ] + ) + ) elif z in range(12, 15): - qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", - 'Route à 1 chaussée', ])) + qs = qs.filter( + Q( + properties__nature__in=[ + "Type autoroutier", + "Route à 2 chaussées", + "Bretelle", + "Route à 1 chaussée", + ] + ) + ) elif self.instance.name == "zone_de_vegetation": if z < 15: - qs = qs.exclude(properties__nature__in=["Verger", 'Vigne', 'Haie', 'Lande ligneuse', 'Peupleraie', 'Bois']) + qs = qs.exclude( + properties__nature__in=[ + "Verger", + "Vigne", + "Haie", + "Lande ligneuse", + "Peupleraie", + "Bois", + ] + ) return qs def get_vector_tile_layer_max_zoom(self): @@ -156,7 +185,11 @@ def get_vector_tile_queryset(self, *args, **kwargs): 19: 0.299, 20: 0.149, } - return qs.annotate(simplified_geom=SimplifyPreserveTopology(Transform("geom", 3857), simplifications.get(z))) + return qs.annotate( + simplified_geom=SimplifyPreserveTopology( + Transform("geom", 3857), simplifications.get(z) + ) + ) class RegionVectorLayer(FullDataFeatureVectorLayer): @@ -199,9 +232,26 @@ def get_vector_tile_queryset(self, z, x, y): if z in range(self.get_vector_tile_layer_min_zoom(), 9): qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) elif z in range(9, 12): - qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", ])) + qs = qs.filter( + Q( + properties__nature__in=[ + "Type autoroutier", + "Route à 2 chaussées", + "Bretelle", + ] + ) + ) elif z in range(12, 15): - qs = qs.filter(Q(properties__nature__in=["Type autoroutier", "Route à 2 chaussées", "Bretelle", 'Route à 1 chaussée',])) + qs = qs.filter( + Q( + properties__nature__in=[ + "Type autoroutier", + "Route à 2 chaussées", + "Bretelle", + "Route à 1 chaussée", + ] + ) + ) return qs diff --git a/test_vectortiles/urls.py b/test_vectortiles/urls.py index 788f232..fc0ee15 100644 --- a/test_vectortiles/urls.py +++ b/test_vectortiles/urls.py @@ -6,92 +6,46 @@ from test_vectortiles.test_app import views router = SimpleRouter() -router.register( - r"features", views.PostGISDRFFeatureViewSet, basename="feature-drf-viewset" -) +router.register(r"features", views.FeatureViewSet, basename="feature-drf-viewset") urlpatterns = [ - # mapbox related urls # feature level path("admin/", admin.site.urls), path( - "features/mapbox/tile///", - views.MapboxFeatureView.as_view(), - name="feature-mapbox", + "features/tile///", + views.FeatureView.as_view(), + name="feature", ), path( - "features/mapbox/tile/{z}/{x}/{y}", - page_not_found, - name="feature-mapbox-pattern", - ), - path( - "features/mapbox/tilejson", - views.MapboxTileJSONFeatureView.as_view(), - name="feature-mapbox-tilejson", - ), - # layer level - path( - "layer//mapbox/tile///", - views.MapboxLayerView.as_view(), - name="layer-mapbox", - ), - path( - "layer//mapbox/tile/{z}/{x}/{y}", - page_not_found, - name="layer-mapbox-pattern", - ), - path( - "layer//mapbox/tilejson", - views.MapboxTileJSONLayerView.as_view(), - name="layer-mapbox-tilejson", - ), - # postgis related urls - # feature level - path( - "features/postgis/tile///", - views.PostGISFeatureView.as_view(), - name="feature-postgis", + "features/tile/date///", + views.FeatureWithDateView.as_view(), + name="feature-date", ), path( - "features/postgis/drf/listview/tile///", - views.PostGISDRFFeatureView.as_view(), - name="feature-postgis-drf", - ), - path( - "features/postgis/tile/manual///", - views.PostGISFeatureViewWithManualVectorTileQuerySet.as_view(), - name="feature-postgis-with-manual-vector-tile-queryset", - ), - path( - "features/postgis/tile/date///", - views.PostGISFeatureWithDateView.as_view(), - name="feature-date-postgis", - ), - path( - "features/postgis/tile/{z}/{x}/{y}", + "features/tile/{z}/{x}/{y}", page_not_found, - name="feature-postgis-pattern", + name="feature-pattern", ), path( - "features/postgis/tilejson", - views.PostGISTileJSONFeatureView.as_view(), - name="feature-postgis-tilejson", + "features/tilejson", + views.TileJSONFeatureView.as_view(), + name="feature-tilejson", ), # layer level path( - "layer//postgis/tile///", - views.PostGISLayerView.as_view(), - name="layer-postgis", + "layer/tile///", + views.LayerView.as_view(), + name="layer", ), path( - "layer//postgis/tile/{z}/{x}/{y}", - page_not_found, - name="layer-postgis-pattern", + "layer/tiles.json", + views.LayerTileJSONView.as_view(), + name="layer-tilejson", ), path( - "layer//postgis/tilejson", - views.PostGISTileJSONLayerView.as_view(), - name="layer-postgis-tilejson", + "layer/tile/{z}/{x}/{y}", + page_not_found, + name="layer-pattern", ), path("", include(router.urls)), path("", views.IndexView.as_view(), name="index"), From 8118759b05463fd231191cb4030e5cdc5aa38070 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 16 Feb 2023 09:31:42 +0100 Subject: [PATCH 04/35] update changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7c702f..8979d88 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,11 @@ CHANGELOG * Drop python 3.6 and Django 2.2 * Add python 3.11 and Django 4.2 +** Breaking changes ** + + * Refactor PostGIS and Python (old named MapBox) backends usage. Use setting to set (default postgis) + * No DetailView anymore. As Tile can have many layers, declare VectorLayer on MTVView (one or many). + * Features * Native MVTRenderer for django-rest-framework From 4f6b9288827d0575264159a335aa537e9c56f42f Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 16 Feb 2023 14:20:19 +0100 Subject: [PATCH 05/35] fix test and README --- README.md | 35 ++++++++++++++++++----------------- setup.py | 7 ++++--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 51a32ec..4c2ee18 100644 --- a/README.md +++ b/README.md @@ -17,15 +17,15 @@ pip install django-vectortiles ``` -* Without any other option, use only vectortiles.postgis +* By default, postgis backend is enabled. * Ensure you have psycopg2 set and installed -#### If you don't want to use Postgis +#### If you don't want to use Postgis and / or PostgreSQL ```bash -pip install django-vectortiles[mapbox] +pip install django-vectortiles[python] ``` * This will incude mapbox_vector_tiles package and its dependencies -* Use only vectortiles.mapbox +* Set VECTOR_TILES_BACKEND to "vectortiles.backends.python" ### Examples @@ -37,31 +37,33 @@ pip install django-vectortiles[mapbox] from django.contrib.gis.db import models -class Layer(models.Model): - name = models.CharField(max_length=250) - - class Feature(models.Model): geom = models.GeometryField(srid=4326) name = models.CharField(max_length=250) - layer = models.ForeignKey(Layer, on_delete=models.CASCADE, related_name='features') ``` -#### Simple model: +#### Simple Example: ```python -# in your view file - -from django.views.generic import ListView -from vectortiles.postgis.views import MVTView +# in a vector_layers.py file +from vectortiles import VectorLayer from yourapp.models import Feature -class FeatureTileView(MVTView, ListView): +class FeatureVectorLayer(VectorLayer): model = Feature vector_tile_layer_name = "features" - vector_tile_fields = ('other_field_to_include', ) + vector_tile_fields = ("name",) + +# in your view file + +from vectortiles.views import MVTView +from yourapp.vector_layers import FeatureVectorLayer + + +class FeatureTileView(MVTView): + layers = [FeatureVectorLayer()] # in your urls file @@ -127,7 +129,6 @@ django-vectortiles can be used with DRF if `renderer_classes` of the view is ove ##### With docker and docker-compose ```bash -docker pull makinacorpus/geodjango:bionic-3.6 docker-compose build # docker-compose up docker-compose run /code/venv/bin/python ./manage.py test diff --git a/setup.py b/setup.py index b43be84..f68ed8a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ 'psycopg2-binary' # for dev and test only. in production, use psycopg2 ] -mapbox = [ +python = [ 'mapbox_vector_tile', 'protobuf<4.21.0', # https://github.com/tilezen/mapbox-vector-tile/issues/113 ] @@ -58,9 +58,10 @@ tests_require=test_require, extras_require={ 'test': test_require, - 'dev': test_require + mapbox + [ + 'dev': test_require + python + [ 'django-debug-toolbar', 'sphinx-rtd-theme' ], - 'mapbox': mapbox, + 'python': python, + 'mapbox': python } ) From 3510a3ba7af1207003ec7b6158e3eacad4e80001 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Wed, 24 May 2023 09:14:13 +0200 Subject: [PATCH 06/35] update test app --- README.md | 49 ++- test_vectortiles/settings.py | 1 + test_vectortiles/test_app/admin.py | 1 + .../0014_fulldatafeature_properties_nature.py | 21 ++ test_vectortiles/test_app/models.py | 1 + .../test_app/templates/index.html | 346 +++++++++++++----- test_vectortiles/test_app/views.py | 47 +-- test_vectortiles/test_app/vt_layers.py | 248 ++++++------- vectortiles/backends/__init__.py | 84 ++--- vectortiles/backends/postgis/__init__.py | 22 +- vectortiles/backends/python/__init__.py | 18 +- vectortiles/mixins.py | 136 +++++-- vectortiles/settings.py | 2 +- vectortiles/tests/test_views.py | 4 +- 14 files changed, 588 insertions(+), 392 deletions(-) create mode 100644 test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py diff --git a/README.md b/README.md index 4c2ee18..9689e55 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ pip install django-vectortiles[python] ### Examples -* assuming you have django.contrib.gis in your INSTALLED_APPS and a gis compatible database backend +* assuming you have ```django.contrib.gis``` in your ```INSTALLED_APPS``` and a gis compatible database backend ```python # in your app models.py @@ -78,42 +78,52 @@ urlpatterns = [ ] ``` -#### Related model: +#### Use TileJSON and multiple domains: ```python # in your view file -from django.views.generic import DetailView -from vectortiles.mixins import BaseVectorTileView -from vectortiles.postgis.views import MVTView -from yourapp.models import Layer +from vectortiles.views import TileJSONView +from django.urls import reverse -class LayerTileView(MVTView, DetailView): - model = Layer - vector_tile_fields = ('other_field_to_include', ) +class FeatureTileJSONView(TileJSONView): + """Simple model TileJSON View""" - def get_vector_tile_layer_name(self): - return self.get_object().name + name = "My features dataset" + attribution = "@JEC Data" + description = "My dataset" - def get_vector_tile_queryset(self): - return self.get_object().features.all() - - def get(self, request, *args, **kwargs): - self.object = self.get_object() - return BaseVectorTileView.get(self,request=request, z=kwargs.get('z'), x=kwargs.get('x'), y=kwargs.get('y')) + def get_tile_url(self): + """ Base MVTView Url used to generates urls in TileJSON in a.tiles.xxxx/{z}/{x}/{y} format """ + return str(reverse("feature-tile", args=(0, 0, 0))).replace("0/0/0", "{z}/{x}/{y}") # in your urls file from django.urls import path from yourapp import views - urlpatterns = [ ... - path('layer//tile///', views.LayerTileView.as_view(), name="layer-tile"), + path('tiles///', views.FeatureTileView.as_view(), name="feature-tile"), + path("feature/tiles.json", views.FeatureTileJSONView.as_view(), name="feature-tilejson"), + ... + + # in your settings file + ALLOWED_HOSTS = [ + "a.tiles.xxxx", + "b.tiles.xxxx", + "c.tiles.xxxx", + ... +] + +VECTOR_TILES_URLS = [ + "https://a.tiles.xxxx", + "https://b.tiles.xxxx", + "https://c.tiles.xxxx", ... ] + ``` #### Usage without PostgreSQL / PostGIS @@ -140,6 +150,7 @@ docker-compose run /code/venv/bin/python ./manage.py test * Install geodjango requirements * Have a postgresql / postgis 2.4+ enabled database * Use a virtualenv + ```bash pip install .[dev] -U ``` diff --git a/test_vectortiles/settings.py b/test_vectortiles/settings.py index dbdb181..d514fc2 100644 --- a/test_vectortiles/settings.py +++ b/test_vectortiles/settings.py @@ -125,5 +125,6 @@ "default": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": "cache", + "TIMEOUT": 60 * 60 * 24 * 60, # nearly 2 months } } diff --git a/test_vectortiles/test_app/admin.py b/test_vectortiles/test_app/admin.py index f05dddb..f07652a 100644 --- a/test_vectortiles/test_app/admin.py +++ b/test_vectortiles/test_app/admin.py @@ -42,3 +42,4 @@ class FulDataFeatureAdmin(OSMGeoAdmin): "layer", ) list_filter = ("layer",) + search_fields = ("id", "properties__cpx_toponyme_de_cours_d_eau") diff --git a/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py b/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py new file mode 100644 index 0000000..3946245 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.17 on 2023-02-20 12:53 + +import django.db.models.expressions +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("test_app", "0013_alter_fulldatalayer_update_datetime"), + ] + + operations = [ + migrations.AddIndex( + model_name="fulldatafeature", + index=models.Index( + django.db.models.expressions.F("properties__nature"), + name="properties_nature", + ), + ), + ] diff --git a/test_vectortiles/test_app/models.py b/test_vectortiles/test_app/models.py index 1ec8bce..cbc7ea5 100644 --- a/test_vectortiles/test_app/models.py +++ b/test_vectortiles/test_app/models.py @@ -57,4 +57,5 @@ class Meta: ordering = ("id",) indexes = [ GinIndex(fields=["properties"], name="feature_properties_gin"), + models.Index(models.F("properties__nature"), name="properties_nature"), ] diff --git a/test_vectortiles/test_app/templates/index.html b/test_vectortiles/test_app/templates/index.html index 67dd260..b9223c5 100644 --- a/test_vectortiles/test_app/templates/index.html +++ b/test_vectortiles/test_app/templates/index.html @@ -13,17 +13,21 @@ } + {# #} +
+{##} + \ No newline at end of file diff --git a/test_vectortiles/test_app/views.py b/test_vectortiles/test_app/views.py index ac68767..4e03c61 100644 --- a/test_vectortiles/test_app/views.py +++ b/test_vectortiles/test_app/views.py @@ -9,18 +9,9 @@ from test_vectortiles.test_app.models import Feature, FullDataLayer from test_vectortiles.test_app.vt_layers import ( - BatimentVectorLayer, - CommuneVectorLayer, - DepartementVectorLayer, - EPCIVectorLayer, FeatureLayerFilteredByDateVectorLayer, FeatureVectorLayer, - FullDataFeatureVectorLayer, - RegionVectorLayer, - SurfaceHydrographiqueVectorLayer, - TerrainDeSportVectorLayer, - TronconRouteVectorLayer, - VoieFerreeVectorLayer, + FullDataFeatureVectorLayer, CityCentroidVectorLayer, ) from vectortiles.mixins import BaseVectorTileView from vectortiles.rest_framework.renderers import MVTRenderer @@ -41,9 +32,9 @@ class FeatureView(FeatureVectorLayers, MVTView): class FeatureTileJSONView(FeatureVectorLayers, TileJSONView): """Simple model TileJSON View""" - vector_tile_tilejson_name = "My feature dataset" - vector_tile_tilejson_attribution = "@IGN - BD Topo 12/2022" - vector_tile_tilejson_description = "My dataset" + name = "My feature dataset" + attribution = "@IGN - BD Topo 12/2022" + description = "My dataset" class MultipleVectorLayers: @@ -51,19 +42,7 @@ def get_layers(self): return [ FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.filter(include_in_tilejson=True) - ] - # return [FullDataFeatureVectorLayer(layer) for layer in FullDataLayer.objects.all()] - return [ - RegionVectorLayer(), - DepartementVectorLayer(), - EPCIVectorLayer(), - CommuneVectorLayer(), - SurfaceHydrographiqueVectorLayer(), - BatimentVectorLayer(), - TronconRouteVectorLayer(), - VoieFerreeVectorLayer(), - TerrainDeSportVectorLayer(), - ] + ] + [CityCentroidVectorLayer()] class LayerView(MultipleVectorLayers, MVTView): @@ -94,9 +73,13 @@ def get_content_status(self, z, x, y): class LayerTileJSONView(MultipleVectorLayers, TileJSONView): """Simple model TileJSON View""" - vector_tile_tilejson_name = "My layers dataset" - vector_tile_tilejson_attribution = "@IGN - BD Topo 12/2022" - vector_tile_tilejson_description = "My dataset" + name = "My layers dataset" + attribution = "@IGN - BD Topo 12/2022" + legend = "https://avatars.githubusercontent.com/u/7448208?s=96&v=4" + description = "My dataset" + center = [1.77, 44.498, 8] + min_zoom = 6 + max_zoom = 18 def get_tile_url(self): return reverse("layer-pattern") @@ -128,9 +111,9 @@ def tile(self, request, z, x, y, *args, **kwargs): class TileJSONFeatureView(TileJSONView): - vector_tile_tilejson_name = "Feature tileset" - vector_tile_tilejson_description = "feature tileset" - vector_tile_tilejson_attribution = "© JEC" + name = "Feature tileset" + description = "feature tileset" + attribution = "© JEC" def get_tile_url(self): return reverse("feature-pattern") diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py index da69853..e6458a3 100644 --- a/test_vectortiles/test_app/vt_layers.py +++ b/test_vectortiles/test_app/vt_layers.py @@ -1,12 +1,13 @@ from hashlib import md5 -from django.contrib.gis.db.models.functions import Centroid, Transform +from django.contrib.gis.db.models.functions import Centroid from django.core.cache import cache -from django.db.models import Q +from django.db.models import FloatField, Q +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast from django.utils.text import slugify -from test_vectortiles.test_app.functions import SimplifyPreserveTopology -from test_vectortiles.test_app.models import Feature, FullDataLayer, Layer +from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer from vectortiles import VectorLayer @@ -36,61 +37,67 @@ def get_tile(self, x, y, z): cache.set(cache_key, tile) return tile - def get_vector_tile_layer_id(self): - return slugify(self.instance.name) - - def get_vector_tile_layer_name(self): + def get_id(self): return slugify(self.instance.name) def get_vector_tile_queryset(self, z, x, y): return self.instance.features.all() - def get_vector_tile_layer_max_zoom(self): + def get_max_zoom(self): return self.instance.max_zoom - def get_vector_tile_layer_min_zoom(self): + def get_min_zoom(self): return self.instance.min_zoom - def get_vector_tile_layer_description(self): + def get_description(self): return self.instance.description -class CityCentroidVectorLayer(FeatureLayerVectorLayer): - vector_tile_geom_name = "centroid" - - def __init__(self): - self.instance = Layer.objects.get(name="Cities") - - def get_vector_tile_layer_min_zoom(self): - return 6 - - def get_vector_tile_layer_id(self): - return "city-centroid" - - def get_vector_tile_layer_name(self): - return "city-centroid" - - def get_vector_tile_queryset(self, *args, **kwargs): - return self.instance.features.all().annotate(centroid=Centroid("geom")) class FeatureLayerFilteredByDateVectorLayer(VectorLayer): - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) + name = "features" + tile_fields = ("name",) def get_vector_tile_queryset(self, *args, **kwargs): return Feature.objects.filter(date="2020-07-07") class FullDataFeatureVectorLayer(VectorLayer): - vector_tile_fields = ("properties",) - def __init__(self, instance): self.instance = instance + def get_tile_fields(self): + if self.instance.name == "troncon_de_route": + return ("nature",) + elif self.instance.name == "batiment": + return ("hauteur",) + elif self.instance.name in ("commune", "commune_centre"): + return ( + "nom", + "population", + "chef_lieu_region", + "chef_lieu_departement", + ) + elif self.instance.name == "troncon_hydrographique": + return ("nom", ) + elif self.instance.name == "parc_ou_reserve": + return ("nom", "nature") + elif self.instance.name == "departement": + return ("nom", "code_insee", "code_insee_region") + elif self.instance.name == "region": + return ("nom", "code_insee",) + elif self.instance.name == "troncon_voie_ferree": + return ("nature", "voies", "etat", "position") + elif self.instance.name == "surface_hydrographique": + return ("nature",) + elif self.instance.name == "terrain_de_sport": + return ("nature",) + return ("properties",) + def get_tile(self, x, y, z): cache_key = md5( - f"{self.get_vector_tile_layer_id()}-{self.instance.update_datetime}-{z}-{x}-{y}".encode() + f"{self.get_id()}-{self.instance.update_datetime}-{z}-{x}-{y}".encode() ).hexdigest() if cache.has_key(cache_key): # NOQA W601 return cache.get(cache_key) @@ -100,16 +107,13 @@ def get_tile(self, x, y, z): cache.set(cache_key, tile, timeout=3600 * 24 * 30) return tile - def get_vector_tile_layer_id(self): - return slugify(self.instance.name) - - def get_vector_tile_layer_name(self): + def get_id(self): return slugify(self.instance.name) def get_vector_tile_queryset(self, z, x, y): qs = self.instance.features.all() if self.instance.name == "troncon_de_route": - if z in range(self.get_vector_tile_layer_min_zoom(), 9): + if z in range(self.get_min_zoom(), 9): qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) elif z in range(9, 12): qs = qs.filter( @@ -132,6 +136,12 @@ def get_vector_tile_queryset(self, z, x, y): ] ) ) + qs = qs.annotate( + nature=KeyTextTransform( + "nature", + "properties", + ) + ) elif self.instance.name == "zone_de_vegetation": if z < 15: qs = qs.exclude( @@ -144,122 +154,78 @@ def get_vector_tile_queryset(self, z, x, y): "Bois", ] ) + elif self.instance.name == "batiment": + qs = qs.annotate( + hauteur=Cast( + KeyTextTransform("hauteur", "properties"), output_field=FloatField() + ) + ) + elif self.instance.name in ("commune", "commune_centre"): + qs = qs.annotate( + nom=KeyTextTransform( + "nom_officiel", + "properties", + ), + population=Cast( + KeyTextTransform("population", "properties"), + output_field=FloatField(), + ), + chef_lieu_region=KeyTextTransform( + "chef_lieu_de_region", + "properties", + ), + chef_lieu_departement=KeyTextTransform( + "chef_lieu_de_departement", + "properties", + ), + ) + elif self.instance.name == "troncon_hydrographique": + qs = qs.exclude(properties__contains={"position_par_rapport_au_sol": "-1", }) + qs = qs.annotate(nom=KeyTextTransform("cpx_toponyme_de_cours_d_eau", "properties")) + elif self.instance.name == "parc_ou_reserve": + qs = qs.annotate(nom=KeyTextTransform("toponyme", "properties"), + nature=KeyTextTransform("nature", "properties")) + elif self.instance.name == "departement": + qs = qs.annotate(nom=KeyTextTransform("nom_officiel", "properties"), + code_insee=KeyTextTransform("code_insee", "properties"), + code_insee_region=KeyTextTransform("code_insee_de_la_region", "properties")) + elif self.instance.name == "region": + qs = qs.annotate(nom=KeyTextTransform("nom_officiel", "properties"), + code_insee=KeyTextTransform("code_insee", "properties")) + elif self.instance.name == "troncon_voie_ferree": + qs = qs.annotate(nature=KeyTextTransform("nature", "properties"), + voies=KeyTextTransform("nombre_de_voies", "properties"), + etat=KeyTextTransform("etat_de_l_objet", "properties"), + position=KeyTextTransform("position_par_rapport_au_sol", "properties")) + elif self.instance.name == "surface_hydrographique": + qs = qs.annotate(nature=KeyTextTransform("nature", "properties")) + elif self.instance.name == "terrain_de_sport": + qs = qs.annotate(nature=KeyTextTransform("nature", "properties")) return qs - def get_vector_tile_layer_max_zoom(self): + def get_max_zoom(self): return self.instance.max_zoom - def get_vector_tile_layer_min_zoom(self): + def get_min_zoom(self): return self.instance.min_zoom - def get_vector_tile_layer_description(self): + def get_description(self): return self.instance.description -class FullDataLayerOptimizeVectorLayer(FullDataFeatureVectorLayer): - vector_tile_geom_name = "simplified_geom" - - def get_vector_tile_queryset(self, *args, **kwargs): - qs = super().get_vector_tile_queryset(*args, **kwargs) - z = args[0] - simplifications = { - 0: 156543, - 1: 78272, - 2: 39136, - 3: 19568, - 4: 9784, - 5: 4892, - 6: 2446, - 7: 1223, - 8: 611.496, - 9: 305.748, - 10: 152.874, - 11: 76.437, - 12: 38.219, - 13: 19.109, - 14: 9.555, - 15: 4.777, - 16: 2.389, - 17: 1.194, - 18: 0.597, - 19: 0.299, - 20: 0.149, - } - return qs.annotate( - simplified_geom=SimplifyPreserveTopology( - Transform("geom", 3857), simplifications.get(z) - ) - ) - +class CityCentroidVectorLayer(FullDataFeatureVectorLayer): + geom_field = "centroid" -class RegionVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="region") - - -class DepartementVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="departement") - - -class EPCIVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="epci") - - -class CommuneVectorLayer(FullDataFeatureVectorLayer): def __init__(self): self.instance = FullDataLayer.objects.get(name="commune") + def get_min_zoom(self): + return 6 -class SurfaceHydrographiqueVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="surface_hydrographique") - - -class VoieFerreeVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="troncon_de_voie_ferree") - - -class TronconRouteVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="troncon_de_route") - - def get_vector_tile_queryset(self, z, x, y): - qs = super().get_vector_tile_queryset(z, x, y) + def get_id(self): + return "commune_centre" - if z in range(self.get_vector_tile_layer_min_zoom(), 9): - qs = qs.filter(properties__contains={"nature": "Type autoroutier"}) - elif z in range(9, 12): - qs = qs.filter( - Q( - properties__nature__in=[ - "Type autoroutier", - "Route à 2 chaussées", - "Bretelle", - ] - ) - ) - elif z in range(12, 15): - qs = qs.filter( - Q( - properties__nature__in=[ - "Type autoroutier", - "Route à 2 chaussées", - "Bretelle", - "Route à 1 chaussée", - ] - ) - ) + def get_vector_tile_queryset(self, *args, **kwargs): + qs = super().get_vector_tile_queryset(*args, **kwargs) + qs = qs.annotate(centroid=Centroid("geom")) return qs - - -class BatimentVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="batiment") - - -class TerrainDeSportVectorLayer(FullDataFeatureVectorLayer): - def __init__(self): - self.instance = FullDataLayer.objects.get(name="terrain_de_sport") diff --git a/vectortiles/backends/__init__.py b/vectortiles/backends/__init__.py index afc6d6a..0c02795 100644 --- a/vectortiles/backends/__init__.py +++ b/vectortiles/backends/__init__.py @@ -8,53 +8,49 @@ class BaseVectorLayerMixin: model = None queryset = None - - vector_tile_queryset = None - vector_tile_queryset_limit = None # if you want to limit element in tile - vector_tile_layer_name = None # name for data layer in vector tile - vector_tile_geom_name = "geom" # geom field to consider in qs - vector_tile_fields = None # other fields to include from qs - vector_tile_generation = ( - None # use mapbox if you installed [mapbox] sub-dependencies - ) - vector_tile_extent = 4096 # define tile extent - vector_tile_buffer = ( - 256 # define buffer around tiles (intersected polygon display without borders) - ) - vector_tile_clip_geom = ( - True # define if feature geometries should be clipped in tile + queryset_limit = None # if you want to limit feature number per tile + + id = "" # id for data layer in vector tile + description = "" + min_zoom = 0 + max_zoom = 22 + geom_field = "geom" # geom field to consider in qs + layer_fields = None # fields description + tile_fields = None # other fields to include from qs + tile_extent = 4096 # define tile extent + tile_buffer = ( + 256 # buffer around tiles (intersected polygon display without borders) ) - vector_tile_layer_id = "" - vector_tile_layer_description = "" - vector_tile_layer_min_zoom = 0 - vector_tile_layer_max_zoom = 22 + clip_geom = True # geometry clipped in tile def check_in_zoom_levels(self, z): - return ( - self.get_vector_tile_layer_min_zoom() - <= z - <= self.get_vector_tile_layer_max_zoom() - ) + return self.get_min_zoom() <= z <= self.get_max_zoom() - def get_vector_tile_layer_id(self): - return self.vector_tile_layer_id + def get_id(self): + return self.id - def get_vector_tile_layer_description(self): - return self.vector_tile_layer_description + def get_description(self): + return self.description - def get_vector_tile_layer_min_zoom(self): - return self.vector_tile_layer_min_zoom + def get_min_zoom(self): + return self.min_zoom - def get_vector_tile_layer_max_zoom(self): - return self.vector_tile_layer_max_zoom + def get_max_zoom(self): + return self.max_zoom + + def get_layer_fields(self): + return self.layer_fields or {} + + def get_tile_fields(self): + return self.tile_fields or () def get_tilejson_vector_layer(self): return { - "id": self.get_vector_tile_layer_id(), - "description": self.get_vector_tile_layer_description(), - "fields": {}, # self.layer_fields(layer), - "minzoom": self.get_vector_tile_layer_min_zoom(), - "maxzoom": self.get_vector_tile_layer_max_zoom(), + "id": self.get_id(), + "description": self.get_description(), + "fields": self.get_layer_fields(), + "minzoom": self.get_min_zoom(), + "maxzoom": self.get_max_zoom(), } @classmethod @@ -81,19 +77,11 @@ def get_queryset(self): def get_vector_tile_queryset(self, *args, **kwargs): """Get feature queryset in tile dynamically""" - return ( - self.vector_tile_queryset - if self.vector_tile_queryset is not None - else self.get_queryset() - ) + return self.get_queryset() - def get_vector_tile_queryset_limit(self): + def get_queryset_limit(self): """Get feature limit by tile dynamically""" - return self.vector_tile_queryset_limit - - def get_vector_tile_layer_name(self): - """Get layer name in tile dynamically""" - return self.vector_tile_layer_name + return self.queryset_limit def get_tile(self, x, y, z): """ diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index 34f6c4c..9c5a0a1 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -15,28 +15,26 @@ def get_tile(self, x, y, z): # keep features intersecting tile filters = { # GeoFuncMixin implicitly transforms to SRID of geom - f"{self.vector_tile_geom_name}__intersects": MakeEnvelope( - xmin, ymin, xmax, ymax, 3857 - ) + f"{self.geom_field}__intersects": MakeEnvelope(xmin, ymin, xmax, ymax, 3857) } features = features.filter(**filters) # annotate prepared geometry for MVT features = features.annotate( geom_prepared=AsMVTGeom( - Transform(self.vector_tile_geom_name, 3857), + Transform(self.geom_field, 3857), MakeEnvelope(xmin, ymin, xmax, ymax, 3857), - self.vector_tile_extent, - self.vector_tile_buffer, - self.vector_tile_clip_geom, + self.tile_extent, + self.tile_buffer, + self.clip_geom, ) ) fields = ( - self.vector_tile_fields + ("geom_prepared",) - if self.vector_tile_fields + self.get_tile_fields() + ("geom_prepared",) + if self.get_tile_fields() else ("geom_prepared",) ) # limit feature number if limit provided - limit = self.get_vector_tile_queryset_limit() + limit = self.get_queryset_limit() if limit: features = features[:limit] # keep values to include in tile (extra included_fields + geometry) @@ -49,8 +47,8 @@ def get_tile(self, x, y, z): sql ), params=[ - self.get_vector_tile_layer_name(), - self.vector_tile_extent, + self.get_id(), + self.tile_extent, "geom_prepared", *params, ], diff --git a/vectortiles/backends/python/__init__.py b/vectortiles/backends/python/__init__.py index fd51711..b46ca63 100644 --- a/vectortiles/backends/python/__init__.py +++ b/vectortiles/backends/python/__init__.py @@ -25,30 +25,30 @@ def get_tile(self, x, y, z): bbox = Polygon.from_bbox((west, south, east, north)) bbox.srid = 3857 - filters = {f"{self.vector_tile_geom_name}__intersects": bbox} + filters = {f"{self.geom_field}__intersects": bbox} features = features.filter(**filters) # limit feature number if limit provided - limit = self.get_vector_tile_queryset_limit() + limit = self.get_queryset_limit() if limit: features = features[:limit] features = features.annotate( clipped=Intersection( - Transform(self.vector_tile_geom_name, 3857), - bbox.buffer(self.pixel_length(z, self.vector_tile_buffer)), + Transform(self.geom_field, 3857), + bbox.buffer(self.pixel_length(z, self.tile_buffer)), ) - if self.vector_tile_clip_geom + if self.clip_geom else F("geom") ) if features: tile = { - "name": self.get_vector_tile_layer_name(), + "name": self.get_id(), "features": [ { "geometry": feature.clipped.wkb.tobytes(), "properties": { key: getattr(feature, key) - for key in self.vector_tile_fields - if self.vector_tile_fields + for key in self.tile_fields + if self.tile_fields }, } for feature in features @@ -57,5 +57,5 @@ def get_tile(self, x, y, z): return mapbox_vector_tile.encode( tile, quantize_bounds=(west, south, east, north), - extents=self.vector_tile_extent, + extents=self.tile_extent, ) diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index 9a4ec6b..c39eaf5 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -11,52 +11,120 @@ def get_layers(self): class BaseTileJSONView(BaseVectorView): - vector_tile_tilejson_name = "" - vector_tile_tilejson_attribution = "" - vector_tile_tilejson_description = "" - - def get_vector_tile_tilejson_min_zoom(self): - min_zoom = min( - layer.get_tilejson_vector_layer()["minzoom"] for layer in self.get_layers() - ) - return min_zoom or 0 - - def get_vector_tile_tilejson_max_zoom(self): - max_zoom = max( - layer.get_tilejson_vector_layer()["maxzoom"] for layer in self.get_layers() - ) - return max_zoom or 22 - - def get_vector_tile_tilejson_attribution(self): - return self.vector_tile_tilejson_attribution - - def get_vector_tile_tilejson_description(self): - return self.vector_tile_tilejson_description - - def get_vector_tile_tilejson_name(self): - return self.vector_tile_tilejson_name + # https://github.com/mapbox/tilejson-spec/tree/master/3.0.0 + name = None + attribution = None + description = None + min_zoom = None # 0 - 30 range. lower than equal than max_zoom. By default, it's min_zoom of all layers. + max_zoom = None # 0 - 30 range. greater than equal than min_zoom. By default, it's max_zoom of all layers. + fill_zoom = None + legend = None + bounds = [-180, -85.05112877980659, 180, 85.0511287798066] + center = None + scheme = "xyz" + version = "1.0.0" + + def __init__(self, *args, **kwargs): + if self.get_min_zoom() > self.get_max_zoom(): + raise ValueError("min_zoom must be lower than equal than max_zoom") + if self.get_max_zoom() < self.get_min_zoom(): + raise ValueError("max_zoom must be greater than equal than min_zoom") + if not (0 <= self.get_min_zoom() <= 30): + raise ValueError("min_zoom should be in range 0 - 30") + if not (0 <= self.get_max_zoom() <= 30): + raise ValueError("max_zoom should be in range 0 - 30") + super().__init__(*args, **kwargs) + + def get_min_zoom(self): + """Get tilejson minzoom from layers or self.min_zoom""" + try: + # minimum zoom level from layers + layers_min_zoom = min(layer.get_min_zoom() for layer in self.get_layers()) + except ValueError: + # case there is no layer defined ... + layers_min_zoom = None + + if self.min_zoom is not None: + # if defined, return max of layers_min_zoom and self.min_zoom + return ( + max(layers_min_zoom, self.min_zoom) + if layers_min_zoom is not None + else self.min_zoom + ) + # if self.min_zoom not defined, return layers_min_zoom or 0 + return layers_min_zoom if layers_min_zoom is not None else 0 + + def get_max_zoom(self): + """Get tilejson manzoom from layers or self.man_zoom""" + try: + # maximum zoom level from layers + layers_max_zoom = min(layer.get_max_zoom() for layer in self.get_layers()) + except ValueError: + # case there is no layer defined ... + layers_max_zoom = None + + if self.max_zoom is not None: + # if defined, return min of layers_max_zoom and self.max_zoom + return ( + min(layers_max_zoom, self.max_zoom) + if layers_max_zoom is not None + else self.max_zoom + ) + # if self.max_zoom not defined, return layers_max_zoom or 30 + return layers_max_zoom if layers_max_zoom is not None else 30 + + def get_fill_zoom(self): + return self.fill_zoom + + def get_attribution(self): + return self.attribution + + def get_legend(self): + return self.legend + + def get_description(self): + return self.description + + def get_name(self): + return self.name + + def get_bounds(self): + return self.bounds + + def get_center(self): + return self.center + + def get_scheme(self): + return self.scheme + + def get_version(self): + return self.version def get_tile_urls(self, tile_url): - if app_settings.VECTOR_TILES_HOSTNAMES: + if app_settings.VECTOR_TILES_URLS: return [ unquote(urljoin(hostname, tile_url)) - for hostname in app_settings.VECTOR_TILES_HOSTNAMES + for hostname in app_settings.VECTOR_TILES_URLS ] else: return [unquote(self.request.build_absolute_uri(tile_url))] def get_tilejson(self, tile_url, version="3.0.0"): - # https://github.com/mapbox/tilejson-spec/tree/3.0/3.0.0 + # https://github.com/mapbox/tilejson-spec/tree/master/3.0.0 return { "tilejson": version, - "name": self.get_vector_tile_tilejson_name(), + "name": self.get_name(), + "description": self.get_description(), + "legend": self.get_legend(), + "attribution": self.get_attribution(), "tiles": self.get_tile_urls(tile_url), - "minzoom": self.get_vector_tile_tilejson_min_zoom(), - "maxzoom": self.get_vector_tile_tilejson_max_zoom(), - # bounds - # center - "attribution": self.get_vector_tile_tilejson_attribution(), - "description": self.get_vector_tile_tilejson_description(), + "minzoom": self.get_min_zoom(), + "maxzoom": self.get_max_zoom(), + "fillzoom": self.get_fill_zoom(), + "bounds": self.get_bounds(), + "center": self.get_center(), + "scheme": self.get_scheme(), + "version": self.get_version(), "vector_layers": [ layer.get_tilejson_vector_layer() for layer in self.get_layers() ], diff --git a/vectortiles/settings.py b/vectortiles/settings.py index a6962c9..236f614 100644 --- a/vectortiles/settings.py +++ b/vectortiles/settings.py @@ -5,4 +5,4 @@ ) VECTOR_TILES_EXTENSION = getattr(settings, "VECTOR_TILES_EXTENSION", "mvt") VECTOR_TILES_BACKEND = "vectortiles.backends.postgis" # to use python backend, set to 'vectortiles.backends.python' -VECTOR_TILES_HOSTNAMES = getattr(settings, "VECTOR_TILES_HOSTNAMES", None) +VECTOR_TILES_URLS = getattr(settings, "VECTOR_TILES_URLS", None) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 2eb3a12..5e15d85 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -31,7 +31,7 @@ def test_num_queries_equals_one(self): def test_layer(self): self.maxDiff = None - response = self.client.get(reverse("layer", args=(self.layer.pk, 0, 0, 0))) + response = self.client.get(reverse("layer", args=(0, 0, 0))) self.assertEqual(response.status_code, 200) content = mapbox_vector_tile.decode(response.content) self.assertDictEqual( @@ -217,7 +217,7 @@ def test_features_with_filtered_date(self): class VectorTileTileJSONTestCase(VectorTileBaseTest): def test_layer(self): self.maxDiff = None - response = self.client.get(reverse("layer-tilejson", args=(self.layer.pk,))) + response = self.client.get(reverse("layer-tilejson")) self.assertEqual(response.status_code, 200) content = response.json() self.assertDictEqual( From 380385d8559c7dca1b7f216d9e7bca697534ef23 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Wed, 24 May 2023 09:24:47 +0200 Subject: [PATCH 07/35] fix linting --- README.md | 10 ++-- docs/conf.py | 3 +- manage.py | 1 + setup.py | 3 +- .../migrations/0006_auto_20230209_0933.py | 1 - .../0007_fulldatafeature_fulldatalayer.py | 1 - .../0008_alter_fulldatafeature_properties.py | 1 - .../0009_alter_fulldatalayer_options.py | 1 - ..._fulldatafeature_feature_properties_gin.py | 1 - .../0011_fulldatalayer_include_in_tilejson.py | 1 - .../0012_fulldatalayer_update_datetime.py | 1 - ...013_alter_fulldatalayer_update_datetime.py | 1 - .../0014_fulldatafeature_properties_nature.py | 1 - test_vectortiles/test_app/views.py | 3 +- test_vectortiles/test_app/vt_layers.py | 53 ++++++++++++------- 15 files changed, 48 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 9689e55..a9d230c 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,10 @@ class Feature(models.Model): #### Simple Example: ```python +from yourapp.models import Feature + # in a vector_layers.py file from vectortiles import VectorLayer -from yourapp.models import Feature class FeatureVectorLayer(VectorLayer): @@ -58,9 +59,10 @@ class FeatureVectorLayer(VectorLayer): # in your view file -from vectortiles.views import MVTView from yourapp.vector_layers import FeatureVectorLayer +from vectortiles.views import MVTView + class FeatureTileView(MVTView): layers = [FeatureVectorLayer()] @@ -70,7 +72,6 @@ class FeatureTileView(MVTView): from django.urls import path from yourapp import views - urlpatterns = [ ... path('tiles///', views.FeatureTileView.as_view(), name="feature-tile"), @@ -83,9 +84,10 @@ urlpatterns = [ ```python # in your view file -from vectortiles.views import TileJSONView from django.urls import reverse +from vectortiles.views import TileJSONView + class FeatureTileJSONView(TileJSONView): """Simple model TileJSON View""" diff --git a/docs/conf.py b/docs/conf.py index 40cd39d..2735928 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,13 +6,14 @@ # -- Path setup -------------------------------------------------------------- +import datetime + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys -import datetime sys.path.insert(0, os.path.abspath('..')) diff --git a/manage.py b/manage.py index 4e29976..7fe8661 100755 --- a/manage.py +++ b/manage.py @@ -4,6 +4,7 @@ import os import sys + def main(): os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_vectortiles.settings') try: diff --git a/setup.py b/setup.py index f68ed8a..030e611 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ #!/usr/bin/env python import os -from setuptools import setup, find_packages + +from setuptools import find_packages, setup HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py b/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py index 73c883b..12ddd40 100644 --- a/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py +++ b/test_vectortiles/test_app/migrations/0006_auto_20230209_0933.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0005_alter_feature_date"), ] diff --git a/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py b/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py index 74b34ae..f01738b 100644 --- a/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py +++ b/test_vectortiles/test_app/migrations/0007_fulldatafeature_fulldatalayer.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0006_auto_20230209_0933"), ] diff --git a/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py b/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py index 9a9e9e5..d6e4bc2 100644 --- a/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py +++ b/test_vectortiles/test_app/migrations/0008_alter_fulldatafeature_properties.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0007_fulldatafeature_fulldatalayer"), ] diff --git a/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py b/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py index fd22ea6..7904d28 100644 --- a/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py +++ b/test_vectortiles/test_app/migrations/0009_alter_fulldatalayer_options.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0008_alter_fulldatafeature_properties"), ] diff --git a/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py b/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py index c103a7a..09664cf 100644 --- a/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py +++ b/test_vectortiles/test_app/migrations/0010_fulldatafeature_feature_properties_gin.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0009_alter_fulldatalayer_options"), ] diff --git a/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py b/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py index 383e842..cb67442 100644 --- a/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py +++ b/test_vectortiles/test_app/migrations/0011_fulldatalayer_include_in_tilejson.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0010_fulldatafeature_feature_properties_gin"), ] diff --git a/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py b/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py index c1fb495..e4149b7 100644 --- a/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py +++ b/test_vectortiles/test_app/migrations/0012_fulldatalayer_update_datetime.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0011_fulldatalayer_include_in_tilejson"), ] diff --git a/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py b/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py index b6cb16b..372b9a0 100644 --- a/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py +++ b/test_vectortiles/test_app/migrations/0013_alter_fulldatalayer_update_datetime.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0012_fulldatalayer_update_datetime"), ] diff --git a/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py b/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py index 3946245..6b721d2 100644 --- a/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py +++ b/test_vectortiles/test_app/migrations/0014_fulldatafeature_properties_nature.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - dependencies = [ ("test_app", "0013_alter_fulldatalayer_update_datetime"), ] diff --git a/test_vectortiles/test_app/views.py b/test_vectortiles/test_app/views.py index 4e03c61..8c45272 100644 --- a/test_vectortiles/test_app/views.py +++ b/test_vectortiles/test_app/views.py @@ -9,9 +9,10 @@ from test_vectortiles.test_app.models import Feature, FullDataLayer from test_vectortiles.test_app.vt_layers import ( + CityCentroidVectorLayer, FeatureLayerFilteredByDateVectorLayer, FeatureVectorLayer, - FullDataFeatureVectorLayer, CityCentroidVectorLayer, + FullDataFeatureVectorLayer, ) from vectortiles.mixins import BaseVectorTileView from vectortiles.rest_framework.renderers import MVTRenderer diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py index e6458a3..b2dbeb2 100644 --- a/test_vectortiles/test_app/vt_layers.py +++ b/test_vectortiles/test_app/vt_layers.py @@ -7,7 +7,7 @@ from django.db.models.functions import Cast from django.utils.text import slugify -from test_vectortiles.test_app.models import Feature, Layer, FullDataLayer +from test_vectortiles.test_app.models import Feature, FullDataLayer, Layer from vectortiles import VectorLayer @@ -53,8 +53,6 @@ def get_description(self): return self.instance.description - - class FeatureLayerFilteredByDateVectorLayer(VectorLayer): name = "features" tile_fields = ("name",) @@ -80,13 +78,16 @@ def get_tile_fields(self): "chef_lieu_departement", ) elif self.instance.name == "troncon_hydrographique": - return ("nom", ) + return ("nom",) elif self.instance.name == "parc_ou_reserve": return ("nom", "nature") elif self.instance.name == "departement": return ("nom", "code_insee", "code_insee_region") elif self.instance.name == "region": - return ("nom", "code_insee",) + return ( + "nom", + "code_insee", + ) elif self.instance.name == "troncon_voie_ferree": return ("nature", "voies", "etat", "position") elif self.instance.name == "surface_hydrographique": @@ -180,23 +181,39 @@ def get_vector_tile_queryset(self, z, x, y): ), ) elif self.instance.name == "troncon_hydrographique": - qs = qs.exclude(properties__contains={"position_par_rapport_au_sol": "-1", }) - qs = qs.annotate(nom=KeyTextTransform("cpx_toponyme_de_cours_d_eau", "properties")) + qs = qs.exclude( + properties__contains={ + "position_par_rapport_au_sol": "-1", + } + ) + qs = qs.annotate( + nom=KeyTextTransform("cpx_toponyme_de_cours_d_eau", "properties") + ) elif self.instance.name == "parc_ou_reserve": - qs = qs.annotate(nom=KeyTextTransform("toponyme", "properties"), - nature=KeyTextTransform("nature", "properties")) + qs = qs.annotate( + nom=KeyTextTransform("toponyme", "properties"), + nature=KeyTextTransform("nature", "properties"), + ) elif self.instance.name == "departement": - qs = qs.annotate(nom=KeyTextTransform("nom_officiel", "properties"), - code_insee=KeyTextTransform("code_insee", "properties"), - code_insee_region=KeyTextTransform("code_insee_de_la_region", "properties")) + qs = qs.annotate( + nom=KeyTextTransform("nom_officiel", "properties"), + code_insee=KeyTextTransform("code_insee", "properties"), + code_insee_region=KeyTextTransform( + "code_insee_de_la_region", "properties" + ), + ) elif self.instance.name == "region": - qs = qs.annotate(nom=KeyTextTransform("nom_officiel", "properties"), - code_insee=KeyTextTransform("code_insee", "properties")) + qs = qs.annotate( + nom=KeyTextTransform("nom_officiel", "properties"), + code_insee=KeyTextTransform("code_insee", "properties"), + ) elif self.instance.name == "troncon_voie_ferree": - qs = qs.annotate(nature=KeyTextTransform("nature", "properties"), - voies=KeyTextTransform("nombre_de_voies", "properties"), - etat=KeyTextTransform("etat_de_l_objet", "properties"), - position=KeyTextTransform("position_par_rapport_au_sol", "properties")) + qs = qs.annotate( + nature=KeyTextTransform("nature", "properties"), + voies=KeyTextTransform("nombre_de_voies", "properties"), + etat=KeyTextTransform("etat_de_l_objet", "properties"), + position=KeyTextTransform("position_par_rapport_au_sol", "properties"), + ) elif self.instance.name == "surface_hydrographique": qs = qs.annotate(nature=KeyTextTransform("nature", "properties")) elif self.instance.name == "terrain_de_sport": From e133b7ee85ecc0862d16af873724287d18652e52 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Wed, 24 May 2023 09:34:25 +0200 Subject: [PATCH 08/35] set version --- vectortiles/VERSION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectortiles/VERSION.md b/vectortiles/VERSION.md index 437b691..547326a 100644 --- a/vectortiles/VERSION.md +++ b/vectortiles/VERSION.md @@ -1 +1 @@ -0.2.0+dev \ No newline at end of file +1.0.0-beta1 \ No newline at end of file From 5e5f0927ae60ea9b0a00171d9fd9db3ba383ef40 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Wed, 24 May 2023 10:37:28 +0200 Subject: [PATCH 09/35] set VERSION.md --- vectortiles/VERSION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectortiles/VERSION.md b/vectortiles/VERSION.md index 547326a..1cff6ec 100644 --- a/vectortiles/VERSION.md +++ b/vectortiles/VERSION.md @@ -1 +1 @@ -1.0.0-beta1 \ No newline at end of file +1.0.0-beta2 \ No newline at end of file From bce65e0f377191e13771fc0740ea2040385290d4 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 25 May 2023 09:21:00 +0200 Subject: [PATCH 10/35] use github actions to upload packages --- .github/workflows/python-publish.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index a6ff8c7..470ec0d 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -21,11 +21,14 @@ jobs: - name: Install dependencies run: | - python -m pip install setuptools wheel twine -U - - name: Build and publish + python -m pip install setuptools wheel -U + + - name: Build packages env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist - twine upload dist/* + python setup.py sdist bdist_wheel + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From 08e68b9904f7a4f54c27b19ab26c2b1aef8e67b0 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 25 May 2023 09:26:24 +0200 Subject: [PATCH 11/35] fix wheel packaging --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 030e611..cf02fe3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ long_description=README, description_content_type="text/markdown", long_description_content_type="text/markdown", - packages=find_packages(), + packages=find_packages(exclude=['*tests', 'test*']), url='https://github.com/submarcos/django-vectortiles.git', classifiers=[ 'Environment :: Web Environment', From 133f32ca03e6d55e3c10636f12179ec3c4b930f1 Mon Sep 17 00:00:00 2001 From: Jordi Castells Date: Wed, 31 May 2023 12:59:59 +0200 Subject: [PATCH 12/35] PostGIS backend uses connection the DB provided by get_queryset. This is made to not assume that the default db is the one always used. --- vectortiles/backends/postgis/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index 9c5a0a1..621fe34 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -1,5 +1,5 @@ from django.contrib.gis.db.models.functions import Transform -from django.db import connection +from django.db import connections from vectortiles.backends import BaseVectorLayerMixin from vectortiles.backends.postgis.functions import AsMVTGeom, MakeEnvelope @@ -41,7 +41,7 @@ def get_tile(self, x, y, z): features = features.values(*fields) # generate MVT sql, params = features.query.sql_with_params() - with connection.cursor() as cursor: + with connections[features.db].cursor() as cursor: cursor.execute( "SELECT ST_ASMVT(subquery.*, %s, %s, %s) FROM ({}) as subquery".format( sql From 725c0b85513f77041ee7a19396233846c879372f Mon Sep 17 00:00:00 2001 From: Jean-Etienne Castagnede Date: Thu, 1 Jun 2023 16:45:18 +0200 Subject: [PATCH 13/35] Update python-publish.yml --- .github/workflows/python-publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 470ec0d..19f9a50 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -7,7 +7,10 @@ on: release: types: [created] workflow_dispatch: - + +permissions: + id-token: write + jobs: deploy: runs-on: ubuntu-latest From c377d5ef5596ebaa92f07dd7ebb8c863f38f5fcd Mon Sep 17 00:00:00 2001 From: Jean-Etienne Castagnede Date: Thu, 1 Jun 2023 16:46:01 +0200 Subject: [PATCH 14/35] Update VERSION.md --- vectortiles/VERSION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectortiles/VERSION.md b/vectortiles/VERSION.md index 1cff6ec..af60a6d 100644 --- a/vectortiles/VERSION.md +++ b/vectortiles/VERSION.md @@ -1 +1 @@ -1.0.0-beta2 \ No newline at end of file +1.0.0-beta3 From 634cae337466df42c71f0b3f7f679d46b63ea0e7 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:08:09 +0200 Subject: [PATCH 15/35] improve v1 --- .env.dist | 5 ++ .github/workflows/python-lint.yml | 8 +- .github/workflows/python-publish.yml | 2 +- .github/workflows/python-test.yml | 19 +++-- Dockerfile | 4 +- README.md | 10 ++- docker-compose.yml | 12 +-- docs/conf.py | 20 ++--- manage.py | 4 +- setup.py | 79 +++++++++---------- test_vectortiles/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/load_data.py | 22 ++++++ .../{settings.py => settings/__init__.py} | 29 ++++--- test_vectortiles/settings/dev.py | 17 ++++ ...re_id_alter_fulldatafeature_id_and_more.py | 33 ++++++++ test_vectortiles/test_app/views.py | 6 +- vectortiles/mixins.py | 16 +++- vectortiles/tests/test_functions.py | 2 +- 19 files changed, 190 insertions(+), 98 deletions(-) create mode 100644 .env.dist create mode 100644 test_vectortiles/management/__init__.py create mode 100644 test_vectortiles/management/commands/__init__.py create mode 100644 test_vectortiles/management/commands/load_data.py rename test_vectortiles/{settings.py => settings/__init__.py} (83%) create mode 100644 test_vectortiles/settings/dev.py create mode 100644 test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py diff --git a/.env.dist b/.env.dist new file mode 100644 index 0000000..a7ccd75 --- /dev/null +++ b/.env.dist @@ -0,0 +1,5 @@ +SECRET_KEY= +POSTGRES_PASSWORD= +POSTGRES_USER=vectortiles +POSTGRES_NAME=vectortiles +POSTGRES_HOST=postgres \ No newline at end of file diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 28e0a64..ff82ed8 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -12,7 +12,7 @@ jobs: flake8: runs-on: ubuntu-latest container: - image: python:3.7 + image: python:3.8 env: LANG: C.UTF-8 steps: @@ -28,7 +28,7 @@ jobs: isort: runs-on: ubuntu-latest container: - image: python:3.7 + image: python:3.8 env: LANG: C.UTF-8 @@ -45,7 +45,7 @@ jobs: black: runs-on: ubuntu-latest container: - image: python:3.7 + image: python:3.8 env: LANG: C.UTF-8 @@ -62,7 +62,7 @@ jobs: doc: runs-on: ubuntu-latest container: - image: python:3.7 + image: python:3.8 env: LANG: C.UTF-8 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 19f9a50..45e9611 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index be50d42..8607684 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -10,18 +10,23 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + env: + LANG: C.UTF-8 + SECRET_KEY: secret-for-ci-only + env: + POSTGRES_PASSWORD: ci_test + POSTGRES_USER: ci_test + POSTGRES_DB: ci_test strategy: matrix: - python-version: ['3.7', '3.8', '3.11'] + python-version: ['3.8', '3.11'] django-version: ['3.2.*', '4.2.*'] psycopg: ['psycopg2-binary'] postgis-image: ['postgis/postgis:10-2.5', 'postgis/postgis:11-2.5', 'postgis/postgis:latest'] exclude: - postgis-image: 'postgis/postgis:11-2.5' django-version: '3.2.*' # test only with 10-2.5 and latest - - python-version: '3.7' - django-version: '4.2.*' # Django 4.2 supports only python >= 3.8 - postgis-image: 'postgis/postgis:10-2.5' django-version: '4.2.*' # Django 4.2 supports only postgres >= 12 - postgis-image: 'postgis/postgis:11-2.5' @@ -36,9 +41,9 @@ jobs: postgres: image: ${{ matrix.postgis-image }} env: - POSTGRES_PASSWORD: travis_ci_test - POSTGRES_USER: travis_ci_test - POSTGRES_DB: travis_ci_test + POSTGRES_PASSWORD: ci_test + POSTGRES_USER: ci_test + POSTGRES_DB: ci_test ports: - 5432:5432 steps: diff --git a/Dockerfile b/Dockerfile index e879707..e773767 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM makinacorpus/geodjango:bionic-3.6 +FROM makinacorpus/geodjango:focal-3.8 RUN mkdir -p /code/src @@ -7,7 +7,7 @@ RUN chown -R django:django /code USER django -RUN python3.6 -m venv /code/venv +RUN python3.8 -m venv /code/venv RUN /code/venv/bin/pip install --no-cache-dir pip setuptools wheel -U COPY . /code/src diff --git a/README.md b/README.md index a9d230c..78b7ac0 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ from vectortiles.views import MVTView class FeatureTileView(MVTView): - layers = [FeatureVectorLayer()] + layer_classes = [FeatureVectorLayer] # in your urls file @@ -110,9 +110,10 @@ urlpatterns = [ path('tiles///', views.FeatureTileView.as_view(), name="feature-tile"), path("feature/tiles.json", views.FeatureTileJSONView.as_view(), name="feature-tilejson"), ... +] - # in your settings file - ALLOWED_HOSTS = [ +# in your settings file +ALLOWED_HOSTS = [ "a.tiles.xxxx", "b.tiles.xxxx", "c.tiles.xxxx", @@ -123,7 +124,6 @@ VECTOR_TILES_URLS = [ "https://a.tiles.xxxx", "https://b.tiles.xxxx", "https://c.tiles.xxxx", - ... ] ``` @@ -140,6 +140,8 @@ django-vectortiles can be used with DRF if `renderer_classes` of the view is ove ##### With docker and docker-compose +Copy ```.env.dist``` to ```.env``` and fill ```SECRET_KEY``` and ```POSTGRES_PASSWORD``` + ```bash docker-compose build # docker-compose up diff --git a/docker-compose.yml b/docker-compose.yml index 06ba92c..6d4fc1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,11 @@ version: "3" services: postgres: - image: postgis/postgis:10-2.5 + image: postgis/postgis:12-2.5 volumes: - postgres:/var/lib/postgresql/data - environment: - - POSTGRES_PASSWORD=travis_ci_test - - POSTGRES_USER=travis_ci_test - - POSTGRES_DB=travis_ci_test + env_file: + - .env ports: - "5432:5432" @@ -17,7 +15,9 @@ services: depends_on: - postgres environment: - - POSTGRES_HOST=postgres + - DJANGO_SETTINGS_MODULE=test_vectortiles.settings.dev + env_file: + - .env volumes: - .:/code/src ports: diff --git a/docs/conf.py b/docs/conf.py index 2735928..221bf4a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,17 +15,17 @@ import os import sys -sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- HERE = os.path.abspath(os.path.dirname(__file__)) -project = 'django-vectortiles' -copyright = f'2021 - {datetime.date.today().year}, Jean-Etienne Castagnede' -author = 'Jean-Etienne Castagnede' +project = "django-vectortiles" +copyright = f"2021 - {datetime.date.today().year}, Jean-Etienne Castagnede" +author = "Jean-Etienne Castagnede" # The full version, including alpha/beta/rc tags -release = open(os.path.join(HERE, '..', 'vectortiles', 'VERSION.md')).read().strip() +release = open(os.path.join(HERE, "..", "vectortiles", "VERSION.md")).read().strip() # -- General configuration --------------------------------------------------- @@ -36,16 +36,16 @@ extensions = [ "sphinx.ext.autodoc", "sphinx_rtd_theme", - 'sphinx.ext.coverage', + "sphinx.ext.coverage", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- @@ -53,9 +53,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/manage.py b/manage.py index 7fe8661..410ea7f 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,7 @@ def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test_vectortiles.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_vectortiles.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -18,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index cf02fe3..4ac7eaf 100644 --- a/setup.py +++ b/setup.py @@ -6,63 +6,58 @@ HERE = os.path.abspath(os.path.dirname(__file__)) -README = open(os.path.join(HERE, 'README.md')).read() +README = open(os.path.join(HERE, "README.md")).read() test_require = [ - 'factory-boy', - 'flake8', - 'isort', - 'black', - 'coverage', - 'djangorestframework', - 'psycopg2-binary' # for dev and test only. in production, use psycopg2 + "factory-boy", + "flake8", + "isort", + "black", + "coverage", + "djangorestframework", + "psycopg2-binary", # for dev and test only. in production, use psycopg2 ] python = [ - 'mapbox_vector_tile', - 'protobuf<4.21.0', # https://github.com/tilezen/mapbox-vector-tile/issues/113 + "mapbox_vector_tile", + "protobuf<4.21.0", # https://github.com/tilezen/mapbox-vector-tile/issues/113 ] setup( - name='django-vectortiles', - version=open(os.path.join(HERE, 'vectortiles', 'VERSION.md')).read().strip(), + name="django-vectortiles", + version=open(os.path.join(HERE, "vectortiles", "VERSION.md")).read().strip(), include_package_data=True, author="Jean-Etienne Castagnede", - description='Django vector tile generation', + description="Django vector tile generation", long_description=README, description_content_type="text/markdown", long_description_content_type="text/markdown", - packages=find_packages(exclude=['*tests', 'test*']), - url='https://github.com/submarcos/django-vectortiles.git', + packages=find_packages(exclude=["*tests", "test*"]), + url="https://github.com/submarcos/django-vectortiles.git", classifiers=[ - 'Environment :: Web Environment', - 'Framework :: Django', - 'Intended Audience :: Developers', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Development Status :: 5 - Production/Stable', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11' - ], - python_requires='>=3.6', - install_requires=[ - 'django', - 'mercantile' + "Environment :: Web Environment", + "Framework :: Django", + "Intended Audience :: Developers", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], + python_requires=">=3.6", + install_requires=["django", "mercantile"], tests_require=test_require, extras_require={ - 'test': test_require, - 'dev': test_require + python + [ - 'django-debug-toolbar', 'sphinx-rtd-theme' - ], - 'python': python, - 'mapbox': python - } + "test": test_require, + "dev": test_require + python + ["django-debug-toolbar", "sphinx-rtd-theme"], + "python": python, + "mapbox": python, + }, ) diff --git a/test_vectortiles/management/__init__.py b/test_vectortiles/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_vectortiles/management/commands/__init__.py b/test_vectortiles/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_vectortiles/management/commands/load_data.py b/test_vectortiles/management/commands/load_data.py new file mode 100644 index 0000000..c77646e --- /dev/null +++ b/test_vectortiles/management/commands/load_data.py @@ -0,0 +1,22 @@ +from django.contrib.gis.gdal import DataSource +from django.contrib.gis.geos import GEOSGeometry, WKTWriter + +from test_vectortiles.test_app.models import FullDataFeature, FullDataLayer + +ds = DataSource("/code/src/bd_topo_46.gpkg") +for layer in ds: + data_layer = FullDataLayer.objects.get_or_create(name=layer.name)[0] + for feature in layer: + try: + geom = feature.geom.geos.transform(4326, clone=True) + if geom.hasz: + wkt_w = WKTWriter() + wkt_w.outdim = 2 # This sets the writer to output 2D WKT + temp = wkt_w.write(geom) + geom = GEOSGeometry(temp) # The 3D geometry + properties = {field: feature.get(field) for field in layer.fields} + data = FullDataFeature.objects.create( + layer=data_layer, geom=geom, properties=properties + ) + except Exception as exc: + print(exc) diff --git a/test_vectortiles/settings.py b/test_vectortiles/settings/__init__.py similarity index 83% rename from test_vectortiles/settings.py rename to test_vectortiles/settings/__init__.py index d514fc2..c2b3601 100644 --- a/test_vectortiles/settings.py +++ b/test_vectortiles/settings/__init__.py @@ -11,24 +11,23 @@ """ import os +from pathlib import Path -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent +PROJECT_DIR = BASE_DIR.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "+10auwvyy9--087ljr2o_-z^mg^@rx)*pe9--eikkn356awcna" +SECRET_KEY = os.getenv("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [ - "*", -] +DEBUG = False +ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "").split(",") # Application definition @@ -40,7 +39,6 @@ "django.contrib.messages", "django.contrib.gis", "django.contrib.staticfiles", - #'debug_toolbar', "vectortiles", "test_vectortiles.test_app", ] @@ -53,7 +51,6 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - #'debug_toolbar.middleware.DebugToolbarMiddleware', ] ROOT_URLCONF = "test_vectortiles.urls" @@ -83,10 +80,10 @@ DATABASES = { "default": { "ENGINE": "django.contrib.gis.db.backends.postgis", - "USER": "travis_ci_test", - "NAME": "travis_ci_test", - "PASSWORD": "travis_ci_test", - "HOST": os.getenv("POSTGRES_HOST", "127.0.0.1"), + "USER": os.getenv("POSTGRES_USER"), + "NAME": os.getenv("POSTGRES_NAME"), + "PASSWORD": os.getenv("POSTGRES_PASSWORD"), + "HOST": os.getenv("POSTGRES_HOST"), } } @@ -124,7 +121,9 @@ CACHES = { "default": { "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "cache", + "LOCATION": PROJECT_DIR / "cache", "TIMEOUT": 60 * 60 * 24 * 60, # nearly 2 months } } + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/test_vectortiles/settings/dev.py b/test_vectortiles/settings/dev.py new file mode 100644 index 0000000..d5fce52 --- /dev/null +++ b/test_vectortiles/settings/dev.py @@ -0,0 +1,17 @@ +from . import * # NOQA + +DEBUG = True + +ALLOWED_HOSTS = [ + "*", +] + +INSTALLED_APPS += [ + "debug_toolbar", +] + +MIDDLEWARE += [ + "debug_toolbar.middleware.DebugToolbarMiddleware", +] + +SHOW_TOOLBAR_CALLBACK = lambda request: True diff --git a/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py b/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py new file mode 100644 index 0000000..335abd7 --- /dev/null +++ b/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.5 on 2023-09-14 13:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('test_app', '0014_fulldatafeature_properties_nature'), + ] + + operations = [ + migrations.AlterField( + model_name='feature', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='fulldatafeature', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='fulldatalayer', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='layer', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/test_vectortiles/test_app/views.py b/test_vectortiles/test_app/views.py index 8c45272..2c8083a 100644 --- a/test_vectortiles/test_app/views.py +++ b/test_vectortiles/test_app/views.py @@ -23,7 +23,7 @@ class FeatureVectorLayers: - layers = [FeatureVectorLayer()] + layer_classes = [FeatureVectorLayer] class FeatureView(FeatureVectorLayers, MVTView): @@ -87,7 +87,7 @@ def get_tile_url(self): class FeatureWithDateView(MVTView): - layers = [FeatureLayerFilteredByDateVectorLayer()] + layer_classes = [FeatureLayerFilteredByDateVectorLayer] class FeatureAPIView(FeatureVectorLayers, MVTAPIView): @@ -96,7 +96,7 @@ class FeatureAPIView(FeatureVectorLayers, MVTAPIView): class FeatureViewSet(BaseVectorTileView, viewsets.ModelViewSet): queryset = Feature.objects.all() - layers = [FeatureVectorLayers()] + layer_classes = [FeatureVectorLayers] @action( detail=False, diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index c39eaf5..73b836c 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -4,10 +4,24 @@ class BaseVectorView: + layer_classes = None layers = None + def get_layer_classes(self): + return self.layer_classes + + def get_layer_class_kwargs(self): + return {} + def get_layers(self): - return self.layers or [] + return ( + self.layers + if self.layers + else [ + layer_class(**self.get_layer_class_kwargs()) + for layer_class in self.get_layer_classes() + ] + ) class BaseTileJSONView(BaseVectorView): diff --git a/vectortiles/tests/test_functions.py b/vectortiles/tests/test_functions.py index 1b8d41d..ada8003 100644 --- a/vectortiles/tests/test_functions.py +++ b/vectortiles/tests/test_functions.py @@ -7,7 +7,7 @@ class MakeEnvelopeTestCase(TestCase): - def test_implicitely_transform_to_base_srid(self): + def test_implicitly_transform_to_base_srid(self): DJANGO_MAJOR = VERSION[0] if DJANGO_MAJOR < 3 or DJANGO_MAJOR >= 4: # superfluous parenthesis for unknown reason From 30353340bf2443fbabfc57f92b5f27ccdde2174d Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:11:24 +0200 Subject: [PATCH 16/35] improve v1 --- ...re_id_alter_fulldatafeature_id_and_more.py | 35 +++++++++++-------- vectortiles/backends/postgis/__init__.py | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py b/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py index 335abd7..a222b94 100644 --- a/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py +++ b/test_vectortiles/test_app/migrations/0015_alter_feature_id_alter_fulldatafeature_id_and_more.py @@ -4,30 +4,37 @@ class Migration(migrations.Migration): - dependencies = [ - ('test_app', '0014_fulldatafeature_properties_nature'), + ("test_app", "0014_fulldatafeature_properties_nature"), ] operations = [ migrations.AlterField( - model_name='feature', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="feature", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='fulldatafeature', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="fulldatafeature", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='fulldatalayer', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="fulldatalayer", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), migrations.AlterField( - model_name='layer', - name='id', - field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + model_name="layer", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), ), ] diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index fd8f7cf..38b3b75 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -55,4 +55,4 @@ def get_tile(self, x, y, z): ) row = cursor.fetchone()[0] # psycopg2 returns memoryview, psycopg returns bytes - return row.tobytes() if type(row) == memoryview else row or b"" + return row.tobytes() if isinstance(row, memoryview) else row or b"" From aa53a675dab190c960e258a12cf2aaf2aee00728 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:39:36 +0200 Subject: [PATCH 17/35] fix workflow --- .github/workflows/python-test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 8607684..41e2ac7 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,10 +14,9 @@ jobs: env: LANG: C.UTF-8 SECRET_KEY: secret-for-ci-only - env: - POSTGRES_PASSWORD: ci_test - POSTGRES_USER: ci_test - POSTGRES_DB: ci_test + POSTGRES_PASSWORD: ci_test + POSTGRES_USER: ci_test + POSTGRES_DB: ci_test strategy: matrix: python-version: ['3.8', '3.11'] From 00746d9705cf833dfd10e04d2777d8c319bb4699 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:43:42 +0200 Subject: [PATCH 18/35] fix workflow --- .github/workflows/python-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 41e2ac7..4131d1d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,6 +14,7 @@ jobs: env: LANG: C.UTF-8 SECRET_KEY: secret-for-ci-only + POSTGRES_HOST: localhost POSTGRES_PASSWORD: ci_test POSTGRES_USER: ci_test POSTGRES_DB: ci_test From a27df4dc5c3a00f38ddf285a47a03ae0c08548a3 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:46:57 +0200 Subject: [PATCH 19/35] fix workflow --- .github/workflows/python-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 4131d1d..9af332a 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -15,6 +15,7 @@ jobs: LANG: C.UTF-8 SECRET_KEY: secret-for-ci-only POSTGRES_HOST: localhost + POSTGRES_NAME: ci_test POSTGRES_PASSWORD: ci_test POSTGRES_USER: ci_test POSTGRES_DB: ci_test @@ -41,6 +42,7 @@ jobs: postgres: image: ${{ matrix.postgis-image }} env: + POSTGRES_NAME: ci_test POSTGRES_PASSWORD: ci_test POSTGRES_USER: ci_test POSTGRES_DB: ci_test From c86d562b6a45a241e9f2855c1a24d09e01a178c8 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:56:06 +0200 Subject: [PATCH 20/35] fix workflow --- vectortiles/mixins.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index 73b836c..d7a3363 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -7,6 +7,11 @@ class BaseVectorView: layer_classes = None layers = None + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if not self.layers: + self.layers = [] + def get_layer_classes(self): return self.layer_classes From e4abb72144cb9eff69b3471ad34deb956313d1f9 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 15:56:31 +0200 Subject: [PATCH 21/35] fix workflow --- vectortiles/mixins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index d7a3363..6a98f36 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -9,8 +9,8 @@ class BaseVectorView: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if not self.layers: - self.layers = [] + if not self.layer_classes: + self.layer_classes = [] def get_layer_classes(self): return self.layer_classes From 8a54b1eeff91d663e0646db1ace1a60d956fc5a5 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:03:39 +0200 Subject: [PATCH 22/35] fix workflow --- test_vectortiles/test_app/vt_layers.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py index b2dbeb2..45bd928 100644 --- a/test_vectortiles/test_app/vt_layers.py +++ b/test_vectortiles/test_app/vt_layers.py @@ -13,14 +13,14 @@ class FeatureVectorLayer(VectorLayer): model = Feature - vector_tile_layer_name = "features" - vector_tile_fields = ("name",) - vector_tile_queryset_limit = 100 + id = "features" + tile_fields = ("name",) + queryset_limit = 100 class FeatureLayerVectorLayer(VectorLayer): model = Layer - vector_tile_fields = ("name",) + tile_fields = ("name",) def __init__(self, instance): self.instance = instance @@ -40,7 +40,7 @@ def get_tile(self, x, y, z): def get_id(self): return slugify(self.instance.name) - def get_vector_tile_queryset(self, z, x, y): + def get_queryset(self, z, x, y): return self.instance.features.all() def get_max_zoom(self): @@ -54,10 +54,10 @@ def get_description(self): class FeatureLayerFilteredByDateVectorLayer(VectorLayer): - name = "features" + id = "features" tile_fields = ("name",) - def get_vector_tile_queryset(self, *args, **kwargs): + def get_queryset(self, *args, **kwargs): return Feature.objects.filter(date="2020-07-07") @@ -111,7 +111,7 @@ def get_tile(self, x, y, z): def get_id(self): return slugify(self.instance.name) - def get_vector_tile_queryset(self, z, x, y): + def get_queryset(self, z, x, y): qs = self.instance.features.all() if self.instance.name == "troncon_de_route": if z in range(self.get_min_zoom(), 9): @@ -242,7 +242,7 @@ def get_min_zoom(self): def get_id(self): return "commune_centre" - def get_vector_tile_queryset(self, *args, **kwargs): - qs = super().get_vector_tile_queryset(*args, **kwargs) + def get_queryset(self, *args, **kwargs): + qs = super().get_queryset(*args, **kwargs) qs = qs.annotate(centroid=Centroid("geom")) return qs From f612b9e71ae3d67622e7bcd81a4837c0b0999b14 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:10:23 +0200 Subject: [PATCH 23/35] fix workflow --- README.md | 2 ++ setup.py | 1 - vectortiles/tests/test_views.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78b7ac0..c317a21 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ urlpatterns = [ from django.urls import reverse from vectortiles.views import TileJSONView +from yourapp.vector_layers import FeatureVectorLayer class FeatureTileJSONView(TileJSONView): @@ -95,6 +96,7 @@ class FeatureTileJSONView(TileJSONView): name = "My features dataset" attribution = "@JEC Data" description = "My dataset" + layer_classes = [FeatureVectorLayer] def get_tile_url(self): """ Base MVTView Url used to generates urls in TileJSON in a.tiles.xxxx/{z}/{x}/{y} format """ diff --git a/setup.py b/setup.py index 4ac7eaf..586db3a 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", "Programming Language :: Python", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 5e15d85..4911ff8 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -3,6 +3,7 @@ from django.urls import reverse from test_vectortiles.test_app.models import Feature, Layer +from test_vectortiles.test_app.vt_layers import FeatureVectorLayer from vectortiles.views import TileJSONView @@ -271,6 +272,7 @@ def test_features(self): def test_tilejson_view_default(self): class TestView(TileJSONView): + layer_classes = [FeatureVectorLayer] tile_url = "test" instance = TestView() From 7b3dbf91378f749e458278ce4cb80e18f3eda102 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:14:49 +0200 Subject: [PATCH 24/35] fix workflow --- docs/usage.rst | 6 +++--- test_vectortiles/test_app/vt_layers.py | 4 +--- vectortiles/backends/__init__.py | 4 ---- vectortiles/backends/postgis/__init__.py | 2 +- vectortiles/backends/python/__init__.py | 2 +- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 1fed1b8..239787d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -69,12 +69,12 @@ Related model view class LayerTileView(MVTView, DetailView): model = Layer - vector_tile_fields = ('name', ) + tile_fields = ('name', ) - def get_vector_tile_layer_name(self): + def get_id(self): return self.get_object().name - def get_vector_tile_queryset(self): + def get_queryset(self): return self.get_object().features.all() def get(self, request, *args, **kwargs): diff --git a/test_vectortiles/test_app/vt_layers.py b/test_vectortiles/test_app/vt_layers.py index 45bd928..970caf5 100644 --- a/test_vectortiles/test_app/vt_layers.py +++ b/test_vectortiles/test_app/vt_layers.py @@ -26,9 +26,7 @@ def __init__(self, instance): self.instance = instance def get_tile(self, x, y, z): - cache_key = md5( - f"{self.get_vector_tile_layer_id()}-{z}-{x}-{y}".encode() - ).hexdigest() + cache_key = md5(f"{self.get_id()}-{z}-{x}-{y}".encode()).hexdigest() if cache.has_key(cache_key): # NOQA W601 return cache.get(cache_key) diff --git a/vectortiles/backends/__init__.py b/vectortiles/backends/__init__.py index 0c02795..5f55a2f 100644 --- a/vectortiles/backends/__init__.py +++ b/vectortiles/backends/__init__.py @@ -75,10 +75,6 @@ def get_queryset(self): return self.queryset return self.model.objects.all() - def get_vector_tile_queryset(self, *args, **kwargs): - """Get feature queryset in tile dynamically""" - return self.get_queryset() - def get_queryset_limit(self): """Get feature limit by tile dynamically""" return self.queryset_limit diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index 38b3b75..dea2947 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -9,7 +9,7 @@ class VectorLayer(BaseVectorLayerMixin): def get_tile(self, x, y, z): if not self.check_in_zoom_levels(z): return b"" - features = self.get_vector_tile_queryset(z, x, y) + features = self.get_queryset(z, x, y) # get tile coordinates from x, y and z xmin, ymin, xmax, ymax = self.get_bounds(x, y, z) # keep features intersecting tile diff --git a/vectortiles/backends/python/__init__.py b/vectortiles/backends/python/__init__.py index b46ca63..99fa199 100644 --- a/vectortiles/backends/python/__init__.py +++ b/vectortiles/backends/python/__init__.py @@ -18,7 +18,7 @@ def get_tile(self, x, y, z): if not self.check_in_zoom_levels(z): return b"" - features = self.get_vector_tile_queryset(z, x, y) + features = self.get_queryset(z, x, y) # get tile coordinates from x, y and z west, south, east, north = self.get_bounds(x, y, z) From 2515bce54ecc6652df483e0288d70b96e0d11f6f Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:20:42 +0200 Subject: [PATCH 25/35] fix workflow --- vectortiles/mixins.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vectortiles/mixins.py b/vectortiles/mixins.py index 6a98f36..4bffae0 100644 --- a/vectortiles/mixins.py +++ b/vectortiles/mixins.py @@ -7,13 +7,8 @@ class BaseVectorView: layer_classes = None layers = None - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if not self.layer_classes: - self.layer_classes = [] - def get_layer_classes(self): - return self.layer_classes + return self.layer_classes or [] def get_layer_class_kwargs(self): return {} From 9c8573903cfef0fb8e5a56e16e9a337eba9b5b25 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:31:46 +0200 Subject: [PATCH 26/35] fix workflow --- vectortiles/tests/test_views.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 4911ff8..70048f6 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -27,7 +27,7 @@ def test_num_queries_equals_one(self): self.maxDiff = None with self.assertNumQueries(1): self.client.get( - reverse("feature-with-manual-vector-tile-queryset", args=(0, 0, 0)) + reverse("feature", args=(0, 0, 0)) ) def test_layer(self): @@ -226,11 +226,16 @@ def test_layer(self): { "attribution": "© JEC", "description": "generated from data", - "maxzoom": 22, + 'bounds': [-180, -85.05112877980659, 180, 85.0511287798066], + 'center': None, + 'fillzoom': None, + 'legend': None, + "maxzoom": 30, + 'scheme': 'xyz', "minzoom": 0, "name": "Layer's features tileset", "tilejson": "3.0.0", - "tiles": ["/layer/2/tile/{z}/{x}/{y}"], + "tiles": ["http://testserver/layer/2/tile/{z}/{x}/{y}"], "vector_layers": [ { "description": "Feature layer", @@ -253,11 +258,17 @@ def test_features(self): { "attribution": "© JEC", "description": "feature tileset", + 'bounds': [-180, -85.05112877980659, 180, 85.0511287798066], + 'center': None, + 'fillzoom': None, + 'legend': None, + "maxzoom": 30, + 'scheme': 'xyz', "maxzoom": 22, "minzoom": 0, "name": "Feature tileset", "tilejson": "3.0.0", - "tiles": ["/features/tile/{z}/{x}/{y}"], + "tiles": ["http://testserver/features/tile/{z}/{x}/{y}"], "vector_layers": [ { "description": "Feature layer", From 87806a7160bbb912a825d78bbfc2f376722a8d56 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:38:09 +0200 Subject: [PATCH 27/35] fix workflow --- vectortiles/tests/test_views.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 70048f6..2660c00 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -264,26 +264,17 @@ def test_features(self): 'legend': None, "maxzoom": 30, 'scheme': 'xyz', - "maxzoom": 22, "minzoom": 0, "name": "Feature tileset", "tilejson": "3.0.0", "tiles": ["http://testserver/features/tile/{z}/{x}/{y}"], "vector_layers": [ - { - "description": "Feature layer", - "fields": {}, - "id": "features", - "maxzoom": 22, - "minzoom": 0, - } ], }, ) def test_tilejson_view_default(self): class TestView(TileJSONView): - layer_classes = [FeatureVectorLayer] tile_url = "test" instance = TestView() From c11867fa373782e1c252c5fb34c385abb275648e Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:42:38 +0200 Subject: [PATCH 28/35] fix workflow --- vectortiles/backends/__init__.py | 4 ++++ vectortiles/backends/postgis/__init__.py | 2 +- vectortiles/backends/python/__init__.py | 2 +- vectortiles/tests/test_views.py | 29 +++++++++++------------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/vectortiles/backends/__init__.py b/vectortiles/backends/__init__.py index 5f55a2f..0c02795 100644 --- a/vectortiles/backends/__init__.py +++ b/vectortiles/backends/__init__.py @@ -75,6 +75,10 @@ def get_queryset(self): return self.queryset return self.model.objects.all() + def get_vector_tile_queryset(self, *args, **kwargs): + """Get feature queryset in tile dynamically""" + return self.get_queryset() + def get_queryset_limit(self): """Get feature limit by tile dynamically""" return self.queryset_limit diff --git a/vectortiles/backends/postgis/__init__.py b/vectortiles/backends/postgis/__init__.py index dea2947..38b3b75 100644 --- a/vectortiles/backends/postgis/__init__.py +++ b/vectortiles/backends/postgis/__init__.py @@ -9,7 +9,7 @@ class VectorLayer(BaseVectorLayerMixin): def get_tile(self, x, y, z): if not self.check_in_zoom_levels(z): return b"" - features = self.get_queryset(z, x, y) + features = self.get_vector_tile_queryset(z, x, y) # get tile coordinates from x, y and z xmin, ymin, xmax, ymax = self.get_bounds(x, y, z) # keep features intersecting tile diff --git a/vectortiles/backends/python/__init__.py b/vectortiles/backends/python/__init__.py index 99fa199..b46ca63 100644 --- a/vectortiles/backends/python/__init__.py +++ b/vectortiles/backends/python/__init__.py @@ -18,7 +18,7 @@ def get_tile(self, x, y, z): if not self.check_in_zoom_levels(z): return b"" - features = self.get_queryset(z, x, y) + features = self.get_vector_tile_queryset(z, x, y) # get tile coordinates from x, y and z west, south, east, north = self.get_bounds(x, y, z) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 2660c00..9a35751 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -3,7 +3,6 @@ from django.urls import reverse from test_vectortiles.test_app.models import Feature, Layer -from test_vectortiles.test_app.vt_layers import FeatureVectorLayer from vectortiles.views import TileJSONView @@ -26,9 +25,7 @@ class VectorTileTestCase(VectorTileBaseTest): def test_num_queries_equals_one(self): self.maxDiff = None with self.assertNumQueries(1): - self.client.get( - reverse("feature", args=(0, 0, 0)) - ) + self.client.get(reverse("feature", args=(0, 0, 0))) def test_layer(self): self.maxDiff = None @@ -226,12 +223,12 @@ def test_layer(self): { "attribution": "© JEC", "description": "generated from data", - 'bounds': [-180, -85.05112877980659, 180, 85.0511287798066], - 'center': None, - 'fillzoom': None, - 'legend': None, + "bounds": [-180, -85.05112877980659, 180, 85.0511287798066], + "center": None, + "fillzoom": None, + "legend": None, "maxzoom": 30, - 'scheme': 'xyz', + "scheme": "xyz", "minzoom": 0, "name": "Layer's features tileset", "tilejson": "3.0.0", @@ -258,18 +255,18 @@ def test_features(self): { "attribution": "© JEC", "description": "feature tileset", - 'bounds': [-180, -85.05112877980659, 180, 85.0511287798066], - 'center': None, - 'fillzoom': None, - 'legend': None, + "bounds": [-180, -85.05112877980659, 180, 85.0511287798066], + "center": None, + "fillzoom": None, + "legend": None, "maxzoom": 30, - 'scheme': 'xyz', + "scheme": "xyz", "minzoom": 0, "name": "Feature tileset", "tilejson": "3.0.0", "tiles": ["http://testserver/features/tile/{z}/{x}/{y}"], - "vector_layers": [ - ], + "vector_layers": [], + "version": "1.0.0", }, ) From 1b333b8908d72832e1c6c6bc85d0cff6436b6838 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:47:44 +0200 Subject: [PATCH 29/35] fix workflow --- vectortiles/tests/test_views.py | 96 --------------------------------- 1 file changed, 96 deletions(-) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 9a35751..3c30e0c 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -92,102 +92,6 @@ def test_features(self): }, ) - def test_layer_2(self): - self.maxDiff = None - response = self.client.get(reverse("layer", args=(0, 0, 0))) - self.assertEqual(response.status_code, 200) - content = mapbox_vector_tile.decode(response.content) - self.assertDictEqual( - content, - { - "features": { - "extent": 4096, - "version": 2, - "features": [ - { - "geometry": {"type": "Point", "coordinates": [2048, 2048]}, - "properties": {"name": "feat1"}, - "id": 0, - "type": 1, - }, - { - "geometry": { - "type": "LineString", - "coordinates": [[2048, 2048], [2059, 2059]], - }, - "properties": {"name": "feat2"}, - "id": 0, - "type": 2, - }, - ], - } - }, - ) - - def test_drf_api_features(self): - self.maxDiff = None - response = self.client.get(reverse("feature-drf", args=(0, 0, 0))) - self.assertEqual(response.status_code, 200) - content = mapbox_vector_tile.decode(response.content) - self.assertDictEqual( - content, - { - "features": { - "extent": 4096, - "version": 2, - "features": [ - { - "geometry": {"type": "Point", "coordinates": [2048, 2048]}, - "properties": {"name": "feat1"}, - "id": 0, - "type": 1, - }, - { - "geometry": { - "type": "LineString", - "coordinates": [[2048, 2048], [2059, 2059]], - }, - "properties": {"name": "feat2"}, - "id": 0, - "type": 2, - }, - ], - } - }, - ) - - def test_drf_viewset_features(self): - self.maxDiff = None - response = self.client.get(reverse("feature-drf-viewset-tile", args=(0, 0, 0))) - self.assertEqual(response.status_code, 200) - content = mapbox_vector_tile.decode(response.content) - self.assertDictEqual( - content, - { - "features": { - "extent": 4096, - "version": 2, - "features": [ - { - "geometry": {"type": "Point", "coordinates": [2048, 2048]}, - "properties": {"name": "feat1"}, - "id": 0, - "type": 1, - }, - { - "geometry": { - "type": "LineString", - "coordinates": [[2048, 2048], [2059, 2059]], - }, - "properties": {"name": "feat2"}, - "id": 0, - "type": 2, - }, - ], - } - }, - ) - def test_features_with_filtered_date(self): self.maxDiff = None response = self.client.get(reverse("feature-date", args=(0, 0, 0))) From 5b1851a13800eb8c4f553c7ef6ab122e0eddac2f Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:52:07 +0200 Subject: [PATCH 30/35] fix workflow --- vectortiles/tests/test_views.py | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index 3c30e0c..fc1f484 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -27,39 +27,6 @@ def test_num_queries_equals_one(self): with self.assertNumQueries(1): self.client.get(reverse("feature", args=(0, 0, 0))) - def test_layer(self): - self.maxDiff = None - response = self.client.get(reverse("layer", args=(0, 0, 0))) - self.assertEqual(response.status_code, 200) - content = mapbox_vector_tile.decode(response.content) - self.assertDictEqual( - content, - { - "features": { - "extent": 4096, - "version": 1, - "features": [ - { - "geometry": {"type": "Point", "coordinates": [2048, 2048]}, - "properties": {"name": "feat1"}, - "id": 0, - "type": 1, - }, - { - "geometry": { - "type": "LineString", - "coordinates": [[2048, 2048], [2059, 2059]], - }, - "properties": {"name": "feat2"}, - "id": 0, - "type": 2, - }, - ], - } - }, - content, - ) - def test_features(self): self.maxDiff = None response = self.client.get(reverse("feature", args=(0, 0, 0))) From b1927905e3f3f8e5057af7f3d7fab20a4cf4eb3b Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 16:58:11 +0200 Subject: [PATCH 31/35] fix workflow --- vectortiles/tests/test_mixins.py | 9 +-------- vectortiles/tests/test_views.py | 32 -------------------------------- 2 files changed, 1 insertion(+), 40 deletions(-) diff --git a/vectortiles/tests/test_mixins.py b/vectortiles/tests/test_mixins.py index db412cf..80812ad 100644 --- a/vectortiles/tests/test_mixins.py +++ b/vectortiles/tests/test_mixins.py @@ -3,15 +3,8 @@ from vectortiles.mixins import BaseTileJSONView, BaseVectorTileView -class BaseVectorTileMixinTestCase(TestCase): - def test_raise_not_implemented(self): - with self.assertRaises(NotImplementedError): - instance = BaseVectorTileView() - instance.get_tile(0, 0, 0) - - class BaseTileJSONMixinTestCase(TestCase): def test_raise_not_implemented(self): with self.assertRaises(NotImplementedError): instance = BaseTileJSONView() - instance.get_vector_layers() + instance.get_layers() diff --git a/vectortiles/tests/test_views.py b/vectortiles/tests/test_views.py index fc1f484..a81cabc 100644 --- a/vectortiles/tests/test_views.py +++ b/vectortiles/tests/test_views.py @@ -84,38 +84,6 @@ def test_features_with_filtered_date(self): class VectorTileTileJSONTestCase(VectorTileBaseTest): - def test_layer(self): - self.maxDiff = None - response = self.client.get(reverse("layer-tilejson")) - self.assertEqual(response.status_code, 200) - content = response.json() - self.assertDictEqual( - content, - { - "attribution": "© JEC", - "description": "generated from data", - "bounds": [-180, -85.05112877980659, 180, 85.0511287798066], - "center": None, - "fillzoom": None, - "legend": None, - "maxzoom": 30, - "scheme": "xyz", - "minzoom": 0, - "name": "Layer's features tileset", - "tilejson": "3.0.0", - "tiles": ["http://testserver/layer/2/tile/{z}/{x}/{y}"], - "vector_layers": [ - { - "description": "Feature layer", - "fields": {}, - "id": "features", - "maxzoom": 22, - "minzoom": 0, - } - ], - }, - ) - def test_features(self): self.maxDiff = None response = self.client.get(reverse("feature-tilejson")) From eb7a43a9fe698783afb6225c4c286175c85da24f Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 14 Sep 2023 17:09:10 +0200 Subject: [PATCH 32/35] fix workflow --- vectortiles/tests/test_mixins.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/vectortiles/tests/test_mixins.py b/vectortiles/tests/test_mixins.py index 80812ad..e69de29 100644 --- a/vectortiles/tests/test_mixins.py +++ b/vectortiles/tests/test_mixins.py @@ -1,10 +0,0 @@ -from django.test import TestCase - -from vectortiles.mixins import BaseTileJSONView, BaseVectorTileView - - -class BaseTileJSONMixinTestCase(TestCase): - def test_raise_not_implemented(self): - with self.assertRaises(NotImplementedError): - instance = BaseTileJSONView() - instance.get_layers() From 2fd44061b06ff6a019498b43e6e81ac325aea094 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Fri, 15 Sep 2023 13:58:41 +0200 Subject: [PATCH 33/35] improve tests --- vectortiles/tests/test_mixins.py | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/vectortiles/tests/test_mixins.py b/vectortiles/tests/test_mixins.py index e69de29..943fb51 100644 --- a/vectortiles/tests/test_mixins.py +++ b/vectortiles/tests/test_mixins.py @@ -0,0 +1,37 @@ +from django.test import TestCase + +from test_vectortiles.test_app.models import Feature +from vectortiles.backends import BaseVectorLayerMixin + + +class BaseVectorLayerMixinTestCase(TestCase): + def test_raise_not_implemented(self): + with self.assertRaises(NotImplementedError): + instance = BaseVectorLayerMixin() + instance.get_tile(0, 0, 0) + + def test_get_description_return_description_by_default(self): + instance = BaseVectorLayerMixin() + self.assertEqual(instance.get_description(), instance.description) + + def test_get_layer_fields_return_fields_by_default(self): + instance = BaseVectorLayerMixin() + self.assertEqual(instance.get_layer_fields(), {}) + + def test_tilejson_vector_layer(self): + instance = BaseVectorLayerMixin() + self.assertDictEqual( + instance.get_tilejson_vector_layer(), + { + "id": instance.get_id(), + "description": instance.get_description(), + "minzoom": instance.get_min_zoom(), + "maxzoom": instance.get_max_zoom(), + "fields": instance.get_layer_fields(), + }, + ) + + def test_queryset_is_used(self): + instance = BaseVectorLayerMixin() + instance.queryset = Feature.objects.all() + self.assertEqual(instance.get_queryset(), instance.queryset) From f699606f2c639fa95f13e3aa8cb43c01a549c295 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Fri, 15 Sep 2023 13:58:51 +0200 Subject: [PATCH 34/35] fix debug toolbar in dev mode --- test_vectortiles/urls.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_vectortiles/urls.py b/test_vectortiles/urls.py index fc0ee15..a2d99d5 100644 --- a/test_vectortiles/urls.py +++ b/test_vectortiles/urls.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.contrib import admin from django.urls import include, path from django.views.defaults import page_not_found @@ -50,3 +51,9 @@ path("", include(router.urls)), path("", views.IndexView.as_view(), name="index"), ] + + +if "debug_toolbar" in settings.INSTALLED_APPS: + import debug_toolbar + + urlpatterns += [path("__debug__/", include(debug_toolbar.urls))] From 8de53f79a1b8ec02eb0571a44bd81b1d185e1716 Mon Sep 17 00:00:00 2001 From: Jean-Etienne Castagnede Date: Thu, 8 Feb 2024 16:38:00 +0100 Subject: [PATCH 35/35] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c317a21..405bb46 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ urlpatterns = [ path("feature/tiles.json", views.FeatureTileJSONView.as_view(), name="feature-tilejson"), ... ] - +] # in your settings file ALLOWED_HOSTS = [ "a.tiles.xxxx",