From 9b801b07e311d0750ab7e3e91dc08e9c6d698e83 Mon Sep 17 00:00:00 2001 From: Michael Kofi Armah Date: Thu, 16 Jan 2025 22:22:19 +0000 Subject: [PATCH] [Integration][Gitlab] | Resolve UserWarning Log (#1332) ## Description **What** A `UserWarning` is displayed in the console: ``` UserWarning: Calling a `list()` method without specifying `get_all=True` or `iterator=True` will return a maximum of 20 items. Your query returned 20 of 30 items. See https://python-gitlab.readthedocs.io/en/v3.15.0/api-usage.html#pagination for more details. If this was done intentionally, then this warning can be supressed by adding the argument `get_all=False` to the `list()` call ``` **Why** - This occurs when calling project.labels.list without specifying parameters that handle pagination. - We need to fetch more than the default 20 items without triggering this warning and potentially missing data. **How** We now leverage the `AsyncFetcher.fetch_batch` utility to make the request. Internally, `fetch_batch` sets `get_all=False` and handles pagination automatically, ensuring we can iterate over all items in batches without hitting the 20-item limit or triggering the warning. --- ## Type of change Please leave one option from the following and delete the rest: - [x] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation)

All tests should be run against the port production environment(using a testing org).

### Core testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync finishes successfully - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Scheduled resync able to abort existing resync and start a new one - [ ] Tested with at least 2 integrations from scratch - [ ] Tested with Kafka and Polling event listeners - [ ] Tested deletion of entities that don't pass the selector ### Integration testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Resync finishes successfully - [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory. - [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved - [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected - [ ] Docs PR link [here](#) ### Preflight checklist - [ ] Handled rate limiting - [ ] Handled pagination - [ ] Implemented the code in async - [ ] Support Multi account ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --- integrations/gitlab/CHANGELOG.md | 8 ++++++++ .../gitlab/gitlab_integration/core/async_fetcher.py | 4 +++- .../gitlab/gitlab_integration/gitlab_service.py | 13 ++++++++++--- integrations/gitlab/pyproject.toml | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index eca6961506..6e8a5e98a8 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.2.21 (2025-01-16) +=================== + +### Bug Fixes + +- Fixed UserWarning log caused by not setting the `get_all` parameter for `project.label.list` request + + 0.2.20 (2025-01-16) =================== diff --git a/integrations/gitlab/gitlab_integration/core/async_fetcher.py b/integrations/gitlab/gitlab_integration/core/async_fetcher.py index 586f58058d..d0527b40d0 100644 --- a/integrations/gitlab/gitlab_integration/core/async_fetcher.py +++ b/integrations/gitlab/gitlab_integration/core/async_fetcher.py @@ -15,6 +15,7 @@ User, GroupMember, ProjectFile, + ProjectLabel, ) from loguru import logger @@ -44,7 +45,6 @@ async def fetch_single( ], *args, ) -> Union[ - RESTObject, RESTObject, ProjectPipelineJob, ProjectPipeline, @@ -52,6 +52,7 @@ async def fetch_single( Project, Group, ProjectFile, + ProjectLabel, ]: with ThreadPoolExecutor() as executor: return await get_event_loop().run_in_executor(executor, fetch_func, *args) @@ -92,6 +93,7 @@ async def fetch_batch( List[RESTObject], List[Dict[str, Any]], GitlabList, + List[ProjectLabel], ] ]: def fetch_page( diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index 0e21033806..5847353177 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -20,6 +20,7 @@ ProjectFile, ProjectPipeline, ProjectPipelineJob, + ProjectLabel, Hook, ) from gitlab_integration.core.async_fetcher import AsyncFetcher @@ -550,9 +551,15 @@ async def get_all_projects(self) -> typing.AsyncIterator[List[Project]]: @classmethod async def async_project_labels_wrapper(cls, project: Project) -> dict[str, Any]: try: - labels = await anyio.to_thread.run_sync(project.labels.list) - serialized_labels = [label.attributes for label in labels] - return {"__labels": serialized_labels} + all_labels = [ + label.attributes + async for labels_batch in AsyncFetcher.fetch_batch( + fetch_func=project.labels.list + ) + for label in typing.cast(List[ProjectLabel], labels_batch) + ] + + return {"__labels": all_labels} except Exception as e: logger.warning( f"Failed to get labels for project={project.path_with_namespace}. error={e}" diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 477a9ccc35..69f25387fc 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.2.20" +version = "0.2.21" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "]