diff --git a/.isort.cfg b/.isort.cfg index 182647cb..f5cb8dc5 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -2,4 +2,4 @@ line_length = 88 multi_line_output = 3 include_trailing_comma = True -known_third_party = PIL,aenum,affine,aioboto3,async_lru,asyncpg,boto3,botocore,cachetools,fastapi,gino,httpx,httpx_auth,mercantile,numpy,pendulum,pydantic,pytest,rasterio,shapely,sqlalchemy,starlette +known_third_party = PIL,aenum,affine,aioboto3,async_lru,asyncpg,attr,boto3,botocore,cachetools,fastapi,gino,httpx,httpx_auth,mercantile,morecantile,numpy,pendulum,pydantic,pydantic_settings,pytest,rasterio,rio_tiler,shapely,sqlalchemy,starlette,titiler diff --git a/Dockerfile b/Dockerfile index 8590d6dd..03931dff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get -y update && apt-get -y --no-install-recommends install \ make gcc libc-dev libgeos-dev musl-dev libpq-dev libffi-dev RUN pip install --upgrade pip && pip install pipenv==v2022.11.30 -RUN pip install titiler.application COPY Pipfile Pipfile COPY Pipfile.lock Pipfile.lock diff --git a/app/main.py b/app/main.py index 53d3b328..90b3fa82 100644 --- a/app/main.py +++ b/app/main.py @@ -1,7 +1,4 @@ -""" - - isort:skip_file -""" +"""isort:skip_file.""" import json import logging @@ -45,7 +42,7 @@ from .routes import wmts from .routes import preview -from .routes.titiler import cog, custom, mosaic, routes as titiler_routes +from .routes.titiler import routes as titiler_routes gunicorn_logger = logging.getLogger("gunicorn.error") logger.handlers = gunicorn_logger.handlers @@ -77,9 +74,15 @@ # titiler routes -app.include_router(cog.router, prefix="/titiler/cog", tags=["Single COG Tiles"]) -app.include_router(mosaic.router, prefix="/titiler/mosaic", tags=["Mosaic Tiles"]) -app.include_router(custom.router, prefix="/titiler/custom", tags=["Custom Tiles"]) +app.include_router( + titiler_routes.cog.router, prefix="/cog/basic", tags=["Single COG Tiles"] +) +app.include_router( + titiler_routes.mosaic.router, prefix="/cog/mosaic", tags=["Mosaic Tiles"] +) +app.include_router( + titiler_routes.custom.router, prefix="/cog/custom", tags=["Custom Tiles"] +) ##################### diff --git a/app/models/enumerators/datasets.py b/app/models/enumerators/datasets.py index 820c0814..e2597853 100644 --- a/app/models/enumerators/datasets.py +++ b/app/models/enumerators/datasets.py @@ -34,3 +34,12 @@ class StaticVectorTileCacheDatasets(str, Enum): _datasets = get_datasets(TileCacheType.static_vector_tile_cache) for _dataset in _datasets: extend_enum(StaticVectorTileCacheDatasets, _dataset, _dataset) + + +class COGDatasets(str, Enum): + __doc__ = "Data API datasets with COG assets" + + +_datasets = get_datasets(TileCacheType.cog) +for _dataset in _datasets: + extend_enum(COGDatasets, _dataset, _dataset) diff --git a/app/models/enumerators/tile_caches.py b/app/models/enumerators/tile_caches.py index a2673e61..5dc24321 100644 --- a/app/models/enumerators/tile_caches.py +++ b/app/models/enumerators/tile_caches.py @@ -5,3 +5,4 @@ class TileCacheType(str, Enum): dynamic_vector_tile_cache = "Dynamic vector tile cache" static_vector_tile_cache = "Static vector tile cache" raster_tile_cache = "Raster tile cache" + cog = "COG" diff --git a/app/models/pydantic/database.py b/app/models/pydantic/database.py index 31759137..95e0ed25 100644 --- a/app/models/pydantic/database.py +++ b/app/models/pydantic/database.py @@ -16,7 +16,7 @@ class DatabaseURL(BaseSettings): host: str = Field("localhost", description="Server host.") port: Optional[Union[str, int]] = Field(None, description="Server access port.") username: Optional[str] = Field(None, alias="user", description="Username") - password: Optional[Secret] = Field(None, description="Password") + password: Optional[Union[Secret, str]] = Field(None, description="Password") database: str = Field(..., description="Database name.") url: Optional[URL] = None model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) diff --git a/app/routes/__init__.py b/app/routes/__init__.py index f6ac7728..fbc7ef00 100644 --- a/app/routes/__init__.py +++ b/app/routes/__init__.py @@ -1,13 +1,15 @@ +import os from typing import Optional, Tuple, Union import mercantile import pendulum -from fastapi import Depends, HTTPException, Path, Query +from fastapi import Depends, HTTPException, Path, Query, Request, status from fastapi.logger import logger from shapely.geometry import box from ..crud.sync_db.tile_cache_assets import get_versions from ..models.enumerators.datasets import ( + COGDatasets, DynamicVectorTileCacheDatasets, RasterTileCacheDatasets, StaticVectorTileCacheDatasets, @@ -19,6 +21,9 @@ DATE_REGEX = r"^\d{4}\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$" VERSION_REGEX = r"^v\d{1,8}(\.\d{1,3}){0,2}?$|^latest$" XYZ_REGEX = r"^\d+(@(2|0.5|0.25)x)?$" +VERSION_REGEX_NO_LATEST = r"^v\d{1,8}(\.\d{1,3}){0,2}?$" + +DATA_LAKE_BUCKET = os.environ.get("DATA_LAKE_BUCKET") def to_bbox(x: int, y: int, z: int) -> Bounds: @@ -137,6 +142,42 @@ async def raster_tile_cache_version_dependency( return dataset, version +async def cog_asset_dependency( + request: Request, + dataset: Optional[COGDatasets] = Query(None, description=COGDatasets.__doc__), # type: ignore + version: Optional[str] = Query( + None, description="Data API dataset version.", regex=VERSION_REGEX_NO_LATEST + ), + url: Optional[str] = Query( + None, + description="Dataset path. This needs to be set if `dataset` and `version` query parameters for a Data API dataset are not set.", + ), +) -> Optional[str]: + + if dataset is None and version is None and url is None: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "Need to pass either `url` or `dataset` and `version` pair for Data API dataset.", + ) + + if dataset and version and url: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "Need to pass either `url` or `dataset` and `version` pair, not both.", + ) + + if dataset and version: + folder: str = ( + f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/epsg-4326/cog" + ) + if "bands" in request.query_params: + return folder + + return f"{folder}/default.tif" + + return url + + async def optional_implementation_dependency( implementation: Optional[str] = Query( None, diff --git a/app/routes/titiler/__init__.py b/app/routes/titiler/__init__.py index cefe88ba..e69de29b 100644 --- a/app/routes/titiler/__init__.py +++ b/app/routes/titiler/__init__.py @@ -1 +0,0 @@ -from .routes import cog, custom, mosaic diff --git a/app/routes/titiler/readers.py b/app/routes/titiler/readers.py index ca5e8a14..22660aee 100644 --- a/app/routes/titiler/readers.py +++ b/app/routes/titiler/readers.py @@ -30,9 +30,7 @@ def _maxzoom(self): def __attrs_post_init__(self): """Get grid bounds.""" - band_url: str = self._get_band_url( - "gfw_integrated_alerts/v20230922/raster/epsg-4326/cog/default" - ) + band_url: str = self._get_band_url("default") with self.reader(band_url) as cog: self.bounds = cog.bounds self.crs = cog.crs diff --git a/app/routes/titiler/routes.py b/app/routes/titiler/routes.py index 5bf5e46b..5eaf3005 100644 --- a/app/routes/titiler/routes.py +++ b/app/routes/titiler/routes.py @@ -8,6 +8,7 @@ from titiler.extensions import cogValidateExtension, cogViewerExtension from titiler.mosaic.factory import MosaicTilerFactory +from ...routes import cog_asset_dependency from .algorithms import IntegratedAlerts from .readers import IntegratedAlertsReader @@ -18,7 +19,6 @@ # print("Debugger attached.") -# Add the `Multiply` algorithm to the default ones algorithms: Algorithms = default_algorithms.register( {"integrated_alerts": IntegratedAlerts} ) @@ -27,21 +27,23 @@ PostProcessParams: Callable = algorithms.dependency custom = MultiBandTilerFactory( - router_prefix="/titiler/custom", + router_prefix="/cog/custom", process_dependency=PostProcessParams, reader=IntegratedAlertsReader, + path_dependency=cog_asset_dependency, ) cog = TilerFactory( - router_prefix="/titiler/cog", + router_prefix="/cog/basic", extensions=[ cogValidateExtension(), cogViewerExtension(), ], + path_dependency=cog_asset_dependency, ) algorithms = AlgorithmFactory() -mosaic = MosaicTilerFactory(router_prefix="/titiler/mosaic") +mosaic = MosaicTilerFactory(router_prefix="/cog/mosaic") # add_exception_handlers(app, DEFAULT_STATUS_CODES) diff --git a/terraform/modules/content_delivery_network/main.tf b/terraform/modules/content_delivery_network/main.tf index 24535121..8be7dc7f 100644 --- a/terraform/modules/content_delivery_network/main.tf +++ b/terraform/modules/content_delivery_network/main.tf @@ -423,6 +423,32 @@ resource "aws_cloudfront_distribution" "tiles" { } } + # send all Titiler requests to tile cache app + ordered_cache_behavior { + allowed_methods = local.methods + cached_methods = local.methods + target_origin_id = "dynamic" + compress = true + path_pattern = "cog/*" + default_ttl = 86400 + max_ttl = 86400 + min_ttl = 0 + smooth_streaming = false + trusted_signers = [] + viewer_protocol_policy = "redirect-to-https" + + forwarded_values { + headers = local.headers + query_string = true + query_string_cache_keys = [] + + cookies { + forward = "none" + whitelisted_names = [] + } + } + } + # Default static vector tiles are stored on S3 # They won't change and can stay in cache for a year # We will set response headers for selected tile caches in S3 if required