GTC-2663: Merge Dev into Staging

.pre-commit-config.yaml

@@ -41,4 +41,4 @@ repos:
     hooks:
     - id: detect-secrets
       args: ['--baseline', '.secrets.baseline'] # run: `pip install detect-secrets` to establish baseline
-      exclude: Pipfile.lock
+      exclude: Pipfile.lock

app/crud/async_db/vector_tiles/__init__.py

@@ -30,9 +30,7 @@ def get_mvt_table(
 
 
 async def get_tile(query: Select, name: str, extent: int) -> VectorTileResponse:
-    """
-    Make SQL query to PostgreSQL and return vector tile in PBF format.
-    """
+    """Make SQL query to PostgreSQL and return vector tile in PBF format."""
     query = _as_vector_tile(query, name, extent)
     return await _get_tile(query)
 
@@ -44,9 +42,10 @@ async def get_aggregated_tile(
     name: str,
     extent: int,
 ) -> VectorTileResponse:
-    """
-    Make SQL query to PostgreSQL and return vector tile in PBF format.
-    This function makes a SQL query that aggregates point features based on proximity.
+    """Make SQL query to PostgreSQL and return vector tile in PBF format.
+
+    This function makes a SQL query that aggregates point features based
+    on proximity.
     """
     query = _group_mvt_table(query, columns, group_by_columns).alias(
         "grouped_mvt_table"
@@ -56,17 +55,13 @@ async def get_aggregated_tile(
 
 
 async def _get_tile(query: Select) -> VectorTileResponse:
-
     logger.debug(query)
     tile = await db.scalar(query)
-
     return VectorTileResponse(content=tile, status_code=200)
 
 
 def _get_bounds(left: float, bottom: float, right: float, top: float) -> Select:
-    """
-    Create bounds query
-    """
+    """Create bounds query."""
     geom = db.text(
         "ST_MakeEnvelope(:left, :bottom, :right, :top, 3857) AS geom"
         # "ST_SetSRID(ST_MakeBox2D(ST_Point(:left, :bottom), ST_Point(:right, :top)),3857) AS geom"
@@ -87,9 +82,7 @@ def _get_mvt_table(
     columns: List[ColumnClause],
     order_by: List[ColumnClause] = [],
 ) -> Select:
-    """
-    Create MVT Geom query
-    """
+    """Create MVT Geom query."""
 
     mvt_geom = db.literal_column(
         f"ST_AsMVTGeom(t.geom_wm, bounds.geom::box2d, {extent}, 0,false)"

app/crud/async_db/vector_tiles/nasa_viirs_fire_alerts.py

@@ -3,79 +3,44 @@
 from sqlalchemy.sql.elements import ColumnClause, TextClause
 
 from ....application import db
-from ....models.enumerators.tile_caches import TileCacheType
+from ....models.enumerators.nasa_viirs_fire_alerts.supported_attributes import (
+    SupportedAttribute,
+)
 from ....models.types import Bounds
 from ....responses import VectorTileResponse
 from ...async_db import vector_tiles
-from ...sync_db.tile_cache_assets import get_attributes, get_latest_version
+from ...sync_db.tile_cache_assets import get_attributes
 from . import get_mvt_table
 
 SCHEMA = "nasa_viirs_fire_alerts"
 
-COLUMNS: List[ColumnClause] = list()
-latest_version: Optional[str] = get_latest_version(
-    SCHEMA, TileCacheType.dynamic_vector_tile_cache
-)
-if latest_version:
-    fields = get_attributes(
-        SCHEMA, latest_version, TileCacheType.dynamic_vector_tile_cache
-    )
-    for field in fields:
-        if field["is_feature_info"]:
-            COLUMNS.append(db.column(field["field_name"]))
-
-
-async def get_tile(
-    version: str, bbox: Bounds, extent: int, filters: List[TextClause]
-) -> VectorTileResponse:
-    """
-    Make SQL query to PostgreSQL and return vector tile in PBF format.
-    """
-    query = get_mvt_table(SCHEMA, version, bbox, extent, COLUMNS, filters)
-    return await vector_tiles.get_tile(query, SCHEMA, extent)
-
 
 async def get_aggregated_tile(
     version: str,
     bbox: Bounds,
     extent: int,
-    attributes: List[str],
+    supported_attributes: List[SupportedAttribute],
     filters: List[TextClause],
 ) -> VectorTileResponse:
-    """
-    Make SQL query to PostgreSQL and return vector tile in PBF format.
-    This function makes a SQL query that aggregates point features based on proximity.
+    """Make SQL query to PostgreSQL and return vector tile in PBF format.
+
+    This function makes a SQL query that aggregates point features based
+    on proximity.
     """
 
-    col_dict = {
-        "latitude": db.literal_column("round(avg(latitude),4)").label("latitude"),
-        "longitude": db.literal_column("round(avg(longitude),4)").label("longitude"),
-        "alert__date": db.literal_column(
-            "mode() WITHIN GROUP (ORDER BY alert__date)"
-        ).label("alert__date"),
-        "alert__time_utc": db.literal_column(
-            "mode() WITHIN GROUP (ORDER BY alert__time_utc)"
-        ).label("alert__time_utc"),
-        "confidence__cat": db.literal_column(
-            "mode() WITHIN GROUP (ORDER BY confidence__cat)"
-        ).label("confidence__cat"),
-        "bright_ti4__K": db.literal_column('round(avg("bright_ti4__K"),3)').label(
-            "bright_ti4__K"
-        ),
-        "bright_ti5__K": db.literal_column('round(avg("bright_ti5__k"),3)').label(
-            "bright_ti5__K"
-        ),
-        "frp__MW": db.literal_column('sum("frp__MW")').label("frp__MW"),
-    }
+    columns: List[ColumnClause] = list()
+    attributes: List[str] = await get_attributes(SCHEMA, version)
+    for attribute in attributes:
+        columns.append(db.column(attribute))
 
-    query = get_mvt_table(SCHEMA, version, bbox, extent, COLUMNS, filters)
+    query = get_mvt_table(SCHEMA, version, bbox, extent, columns, filters)
     columns = [
         db.column("geom"),
         db.literal_column("count(*)").label("count"),
     ]
 
-    for attribute in attributes:
-        columns.append(col_dict[attribute])
+    for attribute in supported_attributes:
+        columns.append(attribute.aggregation_rule)
 
     group_by_columns = [db.column("geom")]
 

app/crud/sync_db/tile_cache_assets.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Dict, List, Optional
 
 import pendulum
 from cachetools import TTLCache, cached
@@ -7,6 +7,7 @@
 
 from ...application import get_synchronous_db
 from ...models.enumerators.tile_caches import TileCacheType
+from ...utils.data_api import get_version_fields
 
 
 @cached(cache=TTLCache(maxsize=1, ttl=900))
@@ -23,7 +24,6 @@ def get_all_tile_caches():
                     versions.version as version,
                     assets.creation_options->'implementation' as implementation,
                     versions.is_latest as is_latest,
-                    assets.fields as fields,
                     assets.creation_options->'min_zoom' as min_zoom,
                     assets.creation_options->'max_zoom' as max_zoom,
                     version_metadata.content_start_date as min_date,
@@ -38,11 +38,12 @@ def get_all_tile_caches():
                    WHERE assets.asset_type IN {str(tuple([e.value for e in list(TileCacheType)])).replace('"', "'")}
                     AND assets.status = 'saved'"""
         ).fetchall()
-    if not rows:
-        # tile_caches = []
-        logger.warning("There are no tile caches registered with the API.")
-    # else:
-    # tile_caches = rows
+
+    if rows is None or len(rows) == 0:
+        logger.warning(
+            "No rows returned. There are no tile caches registered with the API"
+        )
+        rows = []
 
     for row in rows:
         tile_caches[row.asset_type].append(
@@ -51,7 +52,6 @@ def get_all_tile_caches():
                 "version": row.version,
                 "implementation": row.implementation,
                 "is_latest": row.is_latest,
-                "fields": row.fields,
                 "min_zoom": row.min_zoom,
                 "max_zoom": row.max_zoom,
                 "min_date": row.min_date.strftime("%Y-%m-%d")
@@ -124,22 +124,11 @@ def get_latest_versions() -> List[Dict[str, str]]:
     return latest_versions
 
 
-@cached(cache=TTLCache(maxsize=15, ttl=900))
-def get_attributes(dataset: str, version: str, asset_type: str) -> List[Dict[str, Any]]:
-    tile_caches = get_all_tile_caches()
-
-    for tile_cache in tile_caches[asset_type]:
-        # pick the first one that matchs
-        # TODO: fetch the correct one for the current implementation
-        #  needs changes to data-api to assure dynamic vector tile caches
-        #  also have the implementation parameter in the creation options
-        if tile_cache["dataset"] == dataset and tile_cache["version"] == version:
-            return tile_cache["fields"]
-
-    logger.warning(
-        f"Did not find any fields in metadata for {asset_type} of {dataset}.{version}."
-    )
-    return list()
+async def get_attributes(dataset, version):
+    # TODO: fetch the correct one for the current implementation
+    #  needs changes to data-api to assure dynamic vector tile caches
+    #  also have the implementation parameter in the creation options
+    return await get_version_fields(dataset, version)
 
 
 @cached(cache=TTLCache(maxsize=100, ttl=900))

app/models/enumerators/nasa_viirs_fire_alerts/supported_attributes.py

@@ -0,0 +1,54 @@
+from aenum import Enum
+
+from ....application import db
+
+
+def rule(aggregation_expression, label):
+    return db.literal_column(aggregation_expression).label(label)
+
+
+class SupportedAttribute(Enum, init="value aggregation_rule"):  # type: ignore
+
+    LATITUDE = ("latitude", rule("round(avg(latitude),4)", "latitude"))
+    LONGITUDE = ("longitude", rule("round(avg(longitude),4)", "longitude"))
+    ALERT_DATE = (
+        "alert__date",
+        rule("mode() WITHIN GROUP (ORDER BY alert__date)", "alert__date"),
+    )
+    ALERT_TIME_UTC = (
+        "alert__time_utc",
+        rule("mode() WITHIN GROUP (ORDER BY alert__time_utc)", "alert__time_utc"),
+    )
+    CONFIDENCE_CAT = (
+        "confidence__cat",
+        rule("mode() WITHIN GROUP (ORDER BY confidence__cat)", "confidence__cat"),
+    )
+    BRIGHT_TI4_K = (
+        "bright_ti4__K",
+        rule('round(avg("bright_ti4__K"),3)', "bright_ti4__K"),
+    )
+    BRIGHT_TI5_K = (
+        "bright_ti5__K",
+        rule('round(avg("bright_ti5__k"),3)', "bright_ti5__K"),
+    )
+    FRP_MW = ("frp__MW", rule('sum("frp__MW")', "frp__MW"))
+    UMD_TREE_COVER_DENSITY_2000__THRESHOLD = (
+        "umd_tree_cover_density_2000__threshold",
+        rule(
+            'max("umd_tree_cover_density_2000__threshold")',
+            "umd_tree_cover_density_2000__threshold",
+        ),
+    )
+    UMD_TREE_COVER_DENSITY__THRESHOLD = (
+        "umd_tree_cover_density__threshold",
+        rule(
+            'max("umd_tree_cover_density__threshold")',
+            "umd_tree_cover_density__threshold",
+        ),
+    )
+
+    def __str__(self):
+        return self.value
+
+
+default_attributes = [SupportedAttribute.FRP_MW]

app/routes/dynamic_vector_tiles.py

@@ -1,9 +1,10 @@
+"""Dynamic vector tiles are generated on the fly.
+
+The dynamic nature of the service allows users to apply filters using
+query parameters or to change tile resolution using the `@` operator
+after the `y` index
 """
-Dynamic vector tiles are generated on the fly.
-The dynamic nature of the service allows users to apply filters using query parameters
-or to change tile resolution using the `@` operator after the `y` index
-"""
-from typing import Dict, List, Optional, Tuple
+from typing import List, Optional, Tuple
 from uuid import UUID
 
 from asyncpg.exceptions import QueryCanceledError
@@ -16,7 +17,6 @@
 from ..crud.async_db.vector_tiles.filters import geometry_filter
 from ..crud.sync_db.tile_cache_assets import get_attributes
 from ..models.enumerators.geostore import GeostoreOrigin
-from ..models.enumerators.tile_caches import TileCacheType
 from ..models.types import Bounds
 from ..responses import VectorTileResponse
 from . import dynamic_vector_tile_cache_version_dependency, vector_xyz
@@ -49,9 +49,7 @@ async def dynamic_vector_tile(
         "If not specified, all attributes will be shown.",
     ),
 ) -> VectorTileResponse:
-    """
-    Generic dynamic vector tile
-    """
+    """Generic dynamic vector tile."""
     dataset, version = dv
     bbox, _, extent = bbox_z
 
@@ -62,24 +60,17 @@ async def dynamic_vector_tile(
     if geom_filter is not None:
         filters.append(geom_filter)
 
-    fields: List[Dict] = get_attributes(
-        dataset, version, TileCacheType.dynamic_vector_tile_cache
-    )
+    attributes: List[str] = await get_attributes(dataset, version)
 
     # if no attributes specified get all feature info fields
     if not include_attribute:
-        columns: List[ColumnClause] = [
-            db.column(field["field_name"])
-            for field in fields
-            if field["is_feature_info"]
-        ]
-
+        columns: List[ColumnClause] = [db.column(attribute) for attribute in attributes]
     # otherwise run provided list against feature info list and keep common elements
     else:
         columns = [
-            db.column(field["field_name"])
-            for field in fields
-            if field["is_feature_info"] and field["field_name"] in include_attribute
+            db.column(attribute)
+            for attribute in attributes
+            if attribute in include_attribute
         ]
 
     query: Select = get_mvt_table(dataset, version, bbox, extent, columns, filters)