From d90069ee03344b769d711466ff94fd43e2f26554 Mon Sep 17 00:00:00 2001 From: Luca Fabbri Date: Fri, 6 Sep 2024 13:12:30 +0200 Subject: [PATCH 1/3] starred search implementation Currently based on a search by id (multi-valued) --- cads_catalogue_api_service/client.py | 7 +++++-- cads_catalogue_api_service/extensions.py | 4 ++++ cads_catalogue_api_service/search_utils.py | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cads_catalogue_api_service/client.py b/cads_catalogue_api_service/client.py index 1c652f1..b47878e 100644 --- a/cads_catalogue_api_service/client.py +++ b/cads_catalogue_api_service/client.py @@ -474,7 +474,7 @@ def load_catalogue( """Return the whole catalogue as a serialized structure.""" query = session.query(self.collection_table).options(*database.deferred_columns) query_results = search_utils.apply_filters( - session, query, q, None, portals=portals + session, query, q, kw=None, idx=None, portals=portals ).all() all_collections = [ collection_serializer( @@ -489,6 +489,7 @@ def all_datasets( request: fastapi.Request, q: str | None = None, kw: list[str] | None = [], + idx: list[str] | None = [], sortby: extensions.CatalogueSortCriterion = extensions.CatalogueSortCriterion.update_desc, page: int = 0, limit: int = 999, @@ -507,7 +508,9 @@ def all_datasets( search = session.query(self.collection_table).options( *database.deferred_columns ) - search = search_utils.apply_filters(session, search, q, kw, portals=portals) + search = search_utils.apply_filters( + session, search, q, kw, idx, portals=portals + ) count = search.count() search = apply_sorting_and_limit( search=search, q=q, sortby=sortby, page=page, limit=limit diff --git a/cads_catalogue_api_service/extensions.py b/cads_catalogue_api_service/extensions.py index 7d1fb25..0c5adfc 100644 --- a/cads_catalogue_api_service/extensions.py +++ b/cads_catalogue_api_service/extensions.py @@ -38,6 +38,9 @@ def datasets_search( kw: list[str] | None = fastapi.Query( default=[], description="Filter by keyword(s)" ), + idx: list[str] | None = fastapi.Query( + default=[], description="Filter by dataset IDs" + ), sortby: CatalogueSortCriterion = fastapi.Query( default=CatalogueSortCriterion.update_desc ), @@ -53,6 +56,7 @@ def datasets_search( request=request, q=q, kw=kw, + idx=idx, sortby=sortby, page=page, limit=limit, diff --git a/cads_catalogue_api_service/search_utils.py b/cads_catalogue_api_service/search_utils.py index b30a61f..a263c8a 100644 --- a/cads_catalogue_api_service/search_utils.py +++ b/cads_catalogue_api_service/search_utils.py @@ -81,6 +81,7 @@ def apply_filters( search: sa.orm.Query, q: str | None, kw: list | None, + idx: list | None, portals: list[str] | None = None, ): """Apply allowed search filters to the running query. @@ -99,6 +100,9 @@ def apply_filters( if portals: search = search.filter(cads_catalogue.database.Resource.portal.in_(portals)) + if idx: + search = search.filter(cads_catalogue.database.Resource.resource_uid.in_(idx)) + # Faceted search if kw: # Facetes search criteria is to run on OR in the same category, and AND between categories From daf9061fec3f79a45208b0abcf0f959f47232af8 Mon Sep 17 00:00:00 2001 From: Luca Fabbri Date: Mon, 9 Sep 2024 10:25:17 +0200 Subject: [PATCH 2/3] /datasets search: enabled POST method --- cads_catalogue_api_service/extensions.py | 40 ++++++++++++++++++++---- environment.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/cads_catalogue_api_service/extensions.py b/cads_catalogue_api_service/extensions.py index 0c5adfc..586248d 100644 --- a/cads_catalogue_api_service/extensions.py +++ b/cads_catalogue_api_service/extensions.py @@ -15,10 +15,11 @@ # limitations under the License. import enum -from typing import Any +from typing import Annotated, Any import attr import fastapi +import pydantic import stac_fastapi.api import stac_fastapi.types.extension @@ -65,6 +66,32 @@ def datasets_search( ) +class FormData(pydantic.BaseModel): + q: str = "" + kw: list[str] | None = [] + idx: list[str] | None = [] + sortby: CatalogueSortCriterion = CatalogueSortCriterion.update_desc + page: int = 0 + limit: int = config.MAX_LIMIT + search_stats: bool = True + + +def datasets_search_post( + request: fastapi.Request, data: Annotated[FormData, fastapi.Form()] +) -> dict[str, Any]: + """Filter datasets based on search parameters.""" + return datasets_search( + request=request, + q=data.q, + kw=data.kw, + idx=data.idx, + sortby=data.sortby, + page=data.page, + limit=data.limit, + search_stats=data.search_stats, + ) + + @attr.s class DatasetsSearchExtension(stac_fastapi.types.extension.ApiExtension): """Datasets filter extension. @@ -100,13 +127,14 @@ def register(self, app: fastapi.FastAPI) -> None: name="Datasets Search", path="/datasets", methods=["GET"], - # endpoint=stac_fastapi.api.routes.create_async_endpoint( - # datasets_search, - # stac_fastapi.api.models.EmptyRequest, - # self.response_class, - # ), endpoint=datasets_search, ) + self.router.add_api_route( + name="Datasets Search", + path="/datasets", + methods=["POST"], + endpoint=datasets_search_post, + ) app.include_router(self.router, tags=["Datasets Search Extension"]) diff --git a/environment.yml b/environment.yml index b73a53d..dee0d4d 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,7 @@ channels: dependencies: - attrs - brotli-asgi -- fastapi>=0.93.0 +- fastapi>=0.113.0 - httpx - pip - pydantic<2 diff --git a/pyproject.toml b/pyproject.toml index 742a01a..875e6e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "attrs", "cads-catalogue@git+https://github.com/ecmwf-projects/cads-catalogue.git", "cads-common@git+https://github.com/ecmwf-projects/cads-common.git", - "fastapi", + "fastapi>=0.113.0", "python-dateutil", "sqlalchemy>=2.0.9", "stac_fastapi.api", From bf61aa624570f27490a9c60b5bbee5b194d3f356 Mon Sep 17 00:00:00 2001 From: Luca Fabbri Date: Mon, 9 Sep 2024 10:38:21 +0200 Subject: [PATCH 3/3] Moved to canonical JSON post for search --- cads_catalogue_api_service/extensions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cads_catalogue_api_service/extensions.py b/cads_catalogue_api_service/extensions.py index 586248d..1fa5607 100644 --- a/cads_catalogue_api_service/extensions.py +++ b/cads_catalogue_api_service/extensions.py @@ -15,7 +15,7 @@ # limitations under the License. import enum -from typing import Annotated, Any +from typing import Any import attr import fastapi @@ -67,6 +67,8 @@ def datasets_search( class FormData(pydantic.BaseModel): + """Search datasets valid payload.""" + q: str = "" kw: list[str] | None = [] idx: list[str] | None = [] @@ -76,9 +78,7 @@ class FormData(pydantic.BaseModel): search_stats: bool = True -def datasets_search_post( - request: fastapi.Request, data: Annotated[FormData, fastapi.Form()] -) -> dict[str, Any]: +def datasets_search_post(request: fastapi.Request, data: FormData) -> dict[str, Any]: """Filter datasets based on search parameters.""" return datasets_search( request=request,