From f79edd243b02f30ddfea16337ed0c363b5baca41 Mon Sep 17 00:00:00 2001 From: Naveen-Pal Date: Sat, 11 Jan 2025 21:01:47 +0530 Subject: [PATCH 1/5] make inactive and archived indexable | modified tests --- backend/apps/github/common.py | 7 +------ backend/apps/github/models/repository.py | 7 +------ backend/apps/owasp/models/chapter.py | 8 +++----- backend/apps/owasp/models/common.py | 4 ++-- backend/apps/owasp/models/mixins/project.py | 2 ++ backend/apps/owasp/models/project.py | 2 +- backend/tests/owasp/models/chapter_tests.py | 14 ++++---------- backend/tests/owasp/models/common_tests.py | 11 ++++------- 8 files changed, 18 insertions(+), 37 deletions(-) diff --git a/backend/apps/github/common.py b/backend/apps/github/common.py index e2cfc5d14..3ad0503dc 100644 --- a/backend/apps/github/common.py +++ b/backend/apps/github/common.py @@ -46,12 +46,7 @@ def sync_repository(gh_repository, organization=None, user=None): ) # GitHub repository issues. - if ( - not repository.is_archived - and repository.track_issues - and repository.project - and repository.project.track_issues - ): + if repository.track_issues and repository.project and repository.project.track_issues: # Sync open issues for the first run. kwargs = { "direction": "asc", diff --git a/backend/apps/github/models/repository.py b/backend/apps/github/models/repository.py index 6cc3c6234..f2fbeca37 100644 --- a/backend/apps/github/models/repository.py +++ b/backend/apps/github/models/repository.py @@ -104,12 +104,7 @@ def __str__(self): @property def is_indexable(self): """Repositories to index.""" - return ( - not self.is_archived - and not self.is_empty - and not self.is_template - and self.project_set.exists() - ) + return not self.is_empty and not self.is_template and self.project_set.exists() @property def latest_release(self): diff --git a/backend/apps/owasp/models/chapter.py b/backend/apps/owasp/models/chapter.py index 85accdd73..90a3db080 100644 --- a/backend/apps/owasp/models/chapter.py +++ b/backend/apps/owasp/models/chapter.py @@ -72,11 +72,9 @@ def active_chapters_count(): def is_indexable(self): """Chapters to index.""" return ( - self.is_active - and self.latitude is not None + self.latitude is not None and self.longitude is not None and not self.owasp_repository.is_empty - and not self.owasp_repository.is_archived ) def from_github(self, repository): @@ -112,8 +110,8 @@ def generate_geo_location(self): def generate_suggested_location(self, open_ai=None, max_tokens=100): """Generate project summary.""" - if not self.is_active: - return + # if not self.is_active: + # return if not (prompt := Prompt.get_owasp_chapter_suggested_location()): return diff --git a/backend/apps/owasp/models/common.py b/backend/apps/owasp/models/common.py index 7c997aed4..0390d4ee7 100644 --- a/backend/apps/owasp/models/common.py +++ b/backend/apps/owasp/models/common.py @@ -52,7 +52,7 @@ class Meta: @property def is_indexable(self): """Entities to index.""" - return self.is_active and self.has_active_repositories + return self.has_active_repositories @property def github_url(self): @@ -105,7 +105,7 @@ def from_github(self, field_mapping, repository): def generate_summary(self, prompt, open_ai=None, max_tokens=500): """Generate entity summary.""" - if not self.is_active or not prompt: + if not prompt: return open_ai = open_ai or OpenAi() diff --git a/backend/apps/owasp/models/mixins/project.py b/backend/apps/owasp/models/mixins/project.py index 0931c4aee..9046a5c24 100644 --- a/backend/apps/owasp/models/mixins/project.py +++ b/backend/apps/owasp/models/mixins/project.py @@ -101,4 +101,6 @@ def idx_type(self): @property def idx_updated_at(self): """Return updated at for indexing.""" + if not self.updated_at: + return None return self.updated_at.timestamp() diff --git a/backend/apps/owasp/models/project.py b/backend/apps/owasp/models/project.py index 30c2b0a79..7181d9dc7 100644 --- a/backend/apps/owasp/models/project.py +++ b/backend/apps/owasp/models/project.py @@ -140,7 +140,7 @@ def is_tool_type(self): @property def is_indexable(self): """Projects to index.""" - return self.is_active and self.has_active_repositories + return self.has_active_repositories @property def nest_key(self): diff --git a/backend/tests/owasp/models/chapter_tests.py b/backend/tests/owasp/models/chapter_tests.py index 4af0eb79c..7ce0d7a7e 100644 --- a/backend/tests/owasp/models/chapter_tests.py +++ b/backend/tests/owasp/models/chapter_tests.py @@ -85,16 +85,10 @@ def test_generate_suggested_location( assert chapter.suggested_location == (expected_location or "") - if is_active: - mock_open_ai.set_input.assert_called_once_with(geo_string) - mock_open_ai.set_max_tokens.assert_called_once_with(100) - mock_open_ai.set_prompt.assert_called_once_with("Tell me the location") - mock_open_ai.complete.assert_called_once() - else: - mock_open_ai.set_input.assert_not_called() - mock_open_ai.set_max_tokens.assert_not_called() - mock_open_ai.set_prompt.assert_not_called() - mock_open_ai.complete.assert_not_called() + mock_open_ai.set_input.assert_called_once_with(geo_string) + mock_open_ai.set_max_tokens.assert_called_once_with(100) + mock_open_ai.set_prompt.assert_called_once_with("Tell me the location") + mock_open_ai.complete.assert_called_once() @pytest.mark.parametrize( ("name", "key", "expected_str"), diff --git a/backend/tests/owasp/models/common_tests.py b/backend/tests/owasp/models/common_tests.py index 6a747c067..3eb53b412 100644 --- a/backend/tests/owasp/models/common_tests.py +++ b/backend/tests/owasp/models/common_tests.py @@ -13,17 +13,14 @@ class Meta: class TestRepositoryBasedEntityModel: @pytest.mark.parametrize( - ("is_active", "has_active_repositories", "expected"), + ("has_active_repositories", "expected"), [ - (True, True, True), - (True, False, False), - (False, True, False), - (False, False, False), + (True, True), + (False, False), ], ) - def test_is_indexable(self, is_active, has_active_repositories, expected): + def test_is_indexable(self, has_active_repositories, expected): model = EntityModel() - model.is_active = is_active model.has_active_repositories = has_active_repositories assert model.is_indexable == expected From f265afc45776ad30808d92a6899b9dc3251756f3 Mon Sep 17 00:00:00 2001 From: Naveen-Pal Date: Sat, 11 Jan 2025 21:14:11 +0530 Subject: [PATCH 2/5] removed unwanted lines --- backend/apps/github/common.py | 7 ++++++- backend/apps/owasp/models/chapter.py | 4 +--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/apps/github/common.py b/backend/apps/github/common.py index 3ad0503dc..e2cfc5d14 100644 --- a/backend/apps/github/common.py +++ b/backend/apps/github/common.py @@ -46,7 +46,12 @@ def sync_repository(gh_repository, organization=None, user=None): ) # GitHub repository issues. - if repository.track_issues and repository.project and repository.project.track_issues: + if ( + not repository.is_archived + and repository.track_issues + and repository.project + and repository.project.track_issues + ): # Sync open issues for the first run. kwargs = { "direction": "asc", diff --git a/backend/apps/owasp/models/chapter.py b/backend/apps/owasp/models/chapter.py index 90a3db080..1d34ff5ac 100644 --- a/backend/apps/owasp/models/chapter.py +++ b/backend/apps/owasp/models/chapter.py @@ -110,9 +110,7 @@ def generate_geo_location(self): def generate_suggested_location(self, open_ai=None, max_tokens=100): """Generate project summary.""" - # if not self.is_active: - # return - + if not (prompt := Prompt.get_owasp_chapter_suggested_location()): return From fba4820e1ba8b3e41acfc3d5ceefdcb80bea30bd Mon Sep 17 00:00:00 2001 From: Naveen-Pal Date: Sat, 11 Jan 2025 21:42:48 +0530 Subject: [PATCH 3/5] fixed multi-line-summary-second-line --- backend/apps/owasp/models/chapter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/apps/owasp/models/chapter.py b/backend/apps/owasp/models/chapter.py index 1d34ff5ac..a954a659c 100644 --- a/backend/apps/owasp/models/chapter.py +++ b/backend/apps/owasp/models/chapter.py @@ -110,7 +110,6 @@ def generate_geo_location(self): def generate_suggested_location(self, open_ai=None, max_tokens=100): """Generate project summary.""" - if not (prompt := Prompt.get_owasp_chapter_suggested_location()): return From 32383b0485f94fe008bc40232fe7f2ae62103510 Mon Sep 17 00:00:00 2001 From: Naveen-Pal Date: Sun, 12 Jan 2025 23:19:31 +0530 Subject: [PATCH 4/5] adjusted the customRanking rules for inactive/archives --- backend/apps/github/index/repository.py | 4 +++- backend/apps/github/models/mixins/repository.py | 5 +++++ backend/apps/owasp/index/chapter.py | 4 ++++ backend/apps/owasp/index/project.py | 2 ++ backend/apps/owasp/models/managers/chapter.py | 2 -- backend/apps/owasp/models/mixins/chapter.py | 10 ++++++++++ backend/apps/owasp/models/mixins/project.py | 5 +++++ 7 files changed, 29 insertions(+), 3 deletions(-) diff --git a/backend/apps/github/index/repository.py b/backend/apps/github/index/repository.py index a2fb1f458..4cbbf3d42 100644 --- a/backend/apps/github/index/repository.py +++ b/backend/apps/github/index/repository.py @@ -32,11 +32,13 @@ class RepositoryIndex(AlgoliaIndex, IndexBase): "idx_subscribers_count", "idx_top_contributors", "idx_topics", + "idx_is_archived", ) settings = { "minProximity": 4, "customRanking": [ + "asc(idx_is_archived)", "desc(idx_stars_count)", "desc(idx_forks_count)", "desc(idx_pushed_at)", @@ -61,7 +63,7 @@ class RepositoryIndex(AlgoliaIndex, IndexBase): def get_queryset(self): """Get queryset for indexing.""" - return Repository.objects.filter(is_archived=False, is_template=False).prefetch_related( + return Repository.objects.filter(is_template=False).prefetch_related( "repositorycontributor_set" ) diff --git a/backend/apps/github/models/mixins/repository.py b/backend/apps/github/models/mixins/repository.py index ef7f7eacd..9b6e97e79 100644 --- a/backend/apps/github/models/mixins/repository.py +++ b/backend/apps/github/models/mixins/repository.py @@ -115,3 +115,8 @@ def idx_top_contributors(self): def idx_topics(self): """Return topics for indexing.""" return self.topics + + @property + def idx_is_archived(self): + """Return URL for indexing.""" + return self.is_archived diff --git a/backend/apps/owasp/index/chapter.py b/backend/apps/owasp/index/chapter.py index 3a973165d..64e64a404 100644 --- a/backend/apps/owasp/index/chapter.py +++ b/backend/apps/owasp/index/chapter.py @@ -27,6 +27,8 @@ class ChapterIndex(AlgoliaIndex): "idx_tags", "idx_updated_at", "idx_url", + "idx_is_active", + "idx_is_chapter_archived", ) geo_field = "idx_geo_location" @@ -40,6 +42,8 @@ class ChapterIndex(AlgoliaIndex): ], "indexLanguages": ["en"], "customRanking": [ + "desc(idx_is_active)", + "asc(idx_is_chapter_archived)", "asc(idx_created_at)", "desc(idx_updated_at)", ], diff --git a/backend/apps/owasp/index/project.py b/backend/apps/owasp/index/project.py index 7cee8a6bb..3caddbb91 100644 --- a/backend/apps/owasp/index/project.py +++ b/backend/apps/owasp/index/project.py @@ -36,6 +36,7 @@ class ProjectIndex(AlgoliaIndex, IndexBase): "idx_type", "idx_updated_at", "idx_url", + "idx_is_active", ) settings = { @@ -47,6 +48,7 @@ class ProjectIndex(AlgoliaIndex, IndexBase): ], "indexLanguages": ["en"], "customRanking": [ + "desc(idx_is_active)", "desc(idx_level_raw)", "desc(idx_stars_count)", "desc(idx_contributors_count)", diff --git a/backend/apps/owasp/models/managers/chapter.py b/backend/apps/owasp/models/managers/chapter.py index 4598027b7..9365c4da8 100644 --- a/backend/apps/owasp/models/managers/chapter.py +++ b/backend/apps/owasp/models/managers/chapter.py @@ -12,10 +12,8 @@ def get_queryset(self): return ( super() .get_queryset() - .filter(is_active=True) .select_related("owasp_repository") .filter( - owasp_repository__is_archived=False, owasp_repository__is_empty=False, ) ) diff --git a/backend/apps/owasp/models/mixins/chapter.py b/backend/apps/owasp/models/mixins/chapter.py index 6475e6a5b..95905a96c 100644 --- a/backend/apps/owasp/models/mixins/chapter.py +++ b/backend/apps/owasp/models/mixins/chapter.py @@ -60,3 +60,13 @@ def idx_top_contributors(self): def idx_updated_at(self): """Return updated at for indexing.""" return (self.updated_at or self.owasp_repository.updated_at).timestamp() + + @property + def idx_is_active(self): + """Return URL for indexing.""" + return self.is_active + + @property + def idx_is_chapter_archived(self): + """Return URL for indexing.""" + return self.owasp_repository.is_archived diff --git a/backend/apps/owasp/models/mixins/project.py b/backend/apps/owasp/models/mixins/project.py index 9046a5c24..7dd4ca5c1 100644 --- a/backend/apps/owasp/models/mixins/project.py +++ b/backend/apps/owasp/models/mixins/project.py @@ -104,3 +104,8 @@ def idx_updated_at(self): if not self.updated_at: return None return self.updated_at.timestamp() + + @property + def idx_is_active(self): + """Return URL for indexing.""" + return self.is_active From e81dc278fc3570ef6149d0327b3d0d602af2efd0 Mon Sep 17 00:00:00 2001 From: Naveen-Pal Date: Fri, 17 Jan 2025 00:53:23 +0530 Subject: [PATCH 5/5] fixed --- backend/apps/github/index/repository.py | 4 ++-- backend/apps/github/models/mixins/repository.py | 4 ++-- backend/apps/owasp/index/chapter.py | 2 -- backend/apps/owasp/models/mixins/chapter.py | 5 ----- backend/apps/owasp/models/mixins/project.py | 4 +--- 5 files changed, 5 insertions(+), 14 deletions(-) diff --git a/backend/apps/github/index/repository.py b/backend/apps/github/index/repository.py index f136f3ef1..ed8b8f23f 100644 --- a/backend/apps/github/index/repository.py +++ b/backend/apps/github/index/repository.py @@ -32,13 +32,13 @@ class RepositoryIndex(AlgoliaIndex, IndexBase): "idx_subscribers_count", "idx_top_contributors", "idx_topics", - "idx_is_archived", + "idx_is_active", ) settings = { "minProximity": 4, "customRanking": [ - "asc(idx_is_archived)", + "desc(idx_is_active)", "desc(idx_stars_count)", "desc(idx_forks_count)", "desc(idx_pushed_at)", diff --git a/backend/apps/github/models/mixins/repository.py b/backend/apps/github/models/mixins/repository.py index 1c10a2acc..e11148ece 100644 --- a/backend/apps/github/models/mixins/repository.py +++ b/backend/apps/github/models/mixins/repository.py @@ -117,6 +117,6 @@ def idx_topics(self): return self.topics @property - def idx_is_archived(self): + def idx_is_active(self): """Return URL for indexing.""" - return self.is_archived + return self.is_active diff --git a/backend/apps/owasp/index/chapter.py b/backend/apps/owasp/index/chapter.py index 4efe80d38..f6afcf617 100644 --- a/backend/apps/owasp/index/chapter.py +++ b/backend/apps/owasp/index/chapter.py @@ -29,7 +29,6 @@ class ChapterIndex(AlgoliaIndex): "idx_updated_at", "idx_url", "idx_is_active", - "idx_is_chapter_archived", ) geo_field = "idx_geo_location" @@ -44,7 +43,6 @@ class ChapterIndex(AlgoliaIndex): "indexLanguages": ["en"], "customRanking": [ "desc(idx_is_active)", - "asc(idx_is_chapter_archived)", "asc(idx_created_at)", "desc(idx_updated_at)", ], diff --git a/backend/apps/owasp/models/mixins/chapter.py b/backend/apps/owasp/models/mixins/chapter.py index 95905a96c..2def3ea54 100644 --- a/backend/apps/owasp/models/mixins/chapter.py +++ b/backend/apps/owasp/models/mixins/chapter.py @@ -65,8 +65,3 @@ def idx_updated_at(self): def idx_is_active(self): """Return URL for indexing.""" return self.is_active - - @property - def idx_is_chapter_archived(self): - """Return URL for indexing.""" - return self.owasp_repository.is_archived diff --git a/backend/apps/owasp/models/mixins/project.py b/backend/apps/owasp/models/mixins/project.py index da79c0d34..9407167c8 100644 --- a/backend/apps/owasp/models/mixins/project.py +++ b/backend/apps/owasp/models/mixins/project.py @@ -160,9 +160,7 @@ def idx_type(self): @property def idx_updated_at(self): """Return updated at for indexing.""" - if not self.updated_at: - return None - return self.updated_at.timestamp() + return self.updated_at.timestamp() if self.updated_at else "" @property def idx_is_active(self):