From ffbd06a5f1c01586d9604aa179fc9bf5151b2c7f Mon Sep 17 00:00:00 2001 From: Arkadii Yakovets <2201626+arkid15r@users.noreply.github.com> Date: Sun, 12 Jan 2025 19:36:23 -0800 Subject: [PATCH] Extend issue/project/release/repository/user index (#438) * Extend issue/project/release/repository/user index * Address comments --------- Co-authored-by: Kate Golovanova --- .../commands/algolia_update_suggestions.py | 4 +- backend/apps/github/index/repository.py | 2 +- backend/apps/github/index/user.py | 5 + .../migrations/0015_alter_release_author.py | 24 ++++ .../apps/github/models/mixins/repository.py | 8 +- backend/apps/github/models/mixins/user.py | 79 +++++++++++++ backend/apps/github/models/release.py | 13 ++- backend/apps/github/models/repository.py | 12 +- .../github/models/repository_contributor.py | 2 +- backend/apps/github/models/user.py | 15 ++- backend/apps/owasp/index/project.py | 12 +- backend/apps/owasp/models/mixins/project.py | 105 ++++++++++++++---- backend/apps/owasp/models/project.py | 16 +++ 13 files changed, 258 insertions(+), 39 deletions(-) create mode 100644 backend/apps/github/migrations/0015_alter_release_author.py diff --git a/backend/apps/common/management/commands/algolia_update_suggestions.py b/backend/apps/common/management/commands/algolia_update_suggestions.py index 853dc8df2..06f7be945 100644 --- a/backend/apps/common/management/commands/algolia_update_suggestions.py +++ b/backend/apps/common/management/commands/algolia_update_suggestions.py @@ -64,13 +64,13 @@ def handle(self, *args, **kwargs): "facets": [ {"attribute": "idx_key"}, {"attribute": "idx_name"}, - {"attribute": "idx_repository_names"}, + {"attribute": "idx_repositories.name"}, {"attribute": "idx_tags"}, ], "generate": [ ["idx_name"], ["idx_tags"], - ["idx_repository_names"], + ["idx_repositories.name"], ], }, "users": { diff --git a/backend/apps/github/index/repository.py b/backend/apps/github/index/repository.py index a2fb1f458..1cd555ed8 100644 --- a/backend/apps/github/index/repository.py +++ b/backend/apps/github/index/repository.py @@ -25,7 +25,7 @@ class RepositoryIndex(AlgoliaIndex, IndexBase): "idx_license", "idx_name", "idx_open_issues_count", - "idx_project", + "idx_project_key", "idx_pushed_at", "idx_size", "idx_stars_count", diff --git a/backend/apps/github/index/user.py b/backend/apps/github/index/user.py index 19c427a64..07c215ead 100644 --- a/backend/apps/github/index/user.py +++ b/backend/apps/github/index/user.py @@ -18,15 +18,20 @@ class UserIndex(AlgoliaIndex, IndexBase): "idx_avatar_url", "idx_bio", "idx_company", + "idx_contributions", "idx_created_at", "idx_email", "idx_followers_count", "idx_following_count", + "idx_issues_count", + "idx_issues", "idx_key", "idx_location", "idx_login", "idx_name", "idx_public_repositories_count", + "idx_releases_count", + "idx_releases", "idx_title", "idx_updated_at", "idx_url", diff --git a/backend/apps/github/migrations/0015_alter_release_author.py b/backend/apps/github/migrations/0015_alter_release_author.py new file mode 100644 index 000000000..6735cbf04 --- /dev/null +++ b/backend/apps/github/migrations/0015_alter_release_author.py @@ -0,0 +1,24 @@ +# Generated by Django 5.1.4 on 2025-01-13 01:36 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("github", "0014_repository_track_issues"), + ] + + operations = [ + migrations.AlterField( + model_name="release", + name="author", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_releases", + to="github.user", + ), + ), + ] diff --git a/backend/apps/github/models/mixins/repository.py b/backend/apps/github/models/mixins/repository.py index ef7f7eacd..2a6c8300c 100644 --- a/backend/apps/github/models/mixins/repository.py +++ b/backend/apps/github/models/mixins/repository.py @@ -43,7 +43,7 @@ def idx_has_funding_yml(self): @property def idx_key(self): """Return key for indexing.""" - return self.path.lower() + return self.nest_key @property def idx_languages(self): @@ -66,9 +66,9 @@ def idx_open_issues_count(self): return self.open_issues_count @property - def idx_project(self): - """Return project for indexing.""" - return self.project.nest_key if self.project else None + def idx_project_key(self): + """Return project key for indexing.""" + return self.project.nest_key if self.project else "" @property def idx_pushed_at(self): diff --git a/backend/apps/github/models/mixins/user.py b/backend/apps/github/models/mixins/user.py index ac8ab0b99..d5db1636a 100644 --- a/backend/apps/github/models/mixins/user.py +++ b/backend/apps/github/models/mixins/user.py @@ -1,5 +1,11 @@ """GitHub user model mixins for index-related functionality.""" +from apps.github.models.repository_contributor import RepositoryContributor + +ISSUES_LIMIT = 6 +RELEASES_LIMIT = 6 +TOP_REPOSITORY_CONTRIBUTORS_LIMIT = 6 + class UserIndexMixin: """User index mixin.""" @@ -69,6 +75,79 @@ def idx_title(self): """Return title for indexing.""" return self.title + @property + def idx_contributions(self): + """Return contributions for indexing.""" + return [ + { + "contributions_count": rc.contributions_count, + "repository_contributors_count": rc.repository.contributors_count, + "repository_description": rc.repository.description, + "repository_forks_count": rc.repository.forks_count, + "repository_key": rc.repository.key.lower(), + "repository_name": rc.repository.name, + "repository_latest_release": str(rc.repository.latest_release.summary) + if rc.repository.latest_release + else "", + "repository_license": rc.repository.license, + "repository_owner_key": rc.repository.owner.login.lower(), + "repository_stars_count": rc.repository.stars_count, + } + for rc in RepositoryContributor.objects.filter(user=self) + .order_by("-contributions_count") + .select_related("repository")[:TOP_REPOSITORY_CONTRIBUTORS_LIMIT] + ] + + @property + def idx_issues(self): + """Return issues for indexing.""" + return [ + { + "created_at": i.created_at.timestamp(), + "comments_count": i.comments_count, + "number": i.number, + "repository": { + "key": i.repository.key, + "owner_key": i.repository.owner.login, + }, + "title": i.title, + } + for i in self.issues.select_related( + "repository", + "repository__owner", + ).order_by("-created_at")[:ISSUES_LIMIT] + ] + + @property + def idx_issues_count(self): + """Return issues count for indexing.""" + return self.issues.count() + + @property + def idx_releases(self): + """Return releases for indexing.""" + return [ + { + "is_pre_release": r.is_pre_release, + "name": r.name, + "published_at": r.published_at.timestamp(), + "repository": { + "key": r.repository.key, + "owner_key": r.repository.owner.login, + }, + "tag_name": r.tag_name, + } + for r in self.releases.select_related( + "repository", + "repository__owner", + ).order_by("-published_at")[:RELEASES_LIMIT] + ] + + @property + def idx_releases_count(self): + """Return releases count for indexing.""" + return self.releases.count() + @property def idx_updated_at(self): """Return updated at timestamp for indexing.""" diff --git a/backend/apps/github/models/release.py b/backend/apps/github/models/release.py index 360e1ebda..f87822007 100644 --- a/backend/apps/github/models/release.py +++ b/backend/apps/github/models/release.py @@ -26,7 +26,13 @@ class Meta: published_at = models.DateTimeField(verbose_name="Published at") # FKs. - author = models.ForeignKey("github.User", on_delete=models.SET_NULL, blank=True, null=True) + author = models.ForeignKey( + "github.User", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="created_releases", + ) repository = models.ForeignKey( "github.Repository", on_delete=models.SET_NULL, @@ -39,6 +45,11 @@ def __str__(self): """User human readable representation.""" return f"{self.name} by {self.author}" + @property + def summary(self): + """Return release summary.""" + return f"{self.tag_name} on {self.published_at.strftime('%b %d, %Y')}" + @property def is_indexable(self): """Releases to index.""" diff --git a/backend/apps/github/models/repository.py b/backend/apps/github/models/repository.py index 6cc3c6234..f30776642 100644 --- a/backend/apps/github/models/repository.py +++ b/backend/apps/github/models/repository.py @@ -114,7 +114,12 @@ def is_indexable(self): @property def latest_release(self): """Repository latest release.""" - return self.releases.order_by("-created_at").first() + return self.published_releases.order_by("-published_at").first() + + @property + def nest_key(self): + """Return repository Nest key.""" + return f"{self.owner.login}-{self.name}" @property def path(self): @@ -126,6 +131,11 @@ def project(self): """Return project.""" return self.project_set.first() + @property + def published_releases(self): + """Return published releases.""" + return self.releases.filter(is_draft=False, published_at__isnull=False) + @property def top_languages(self): """Return a list of top used languages.""" diff --git a/backend/apps/github/models/repository_contributor.py b/backend/apps/github/models/repository_contributor.py index e92069df9..f4be33f2f 100644 --- a/backend/apps/github/models/repository_contributor.py +++ b/backend/apps/github/models/repository_contributor.py @@ -5,7 +5,7 @@ from apps.common.models import BulkSaveModel, TimestampedModel -TOP_CONTRIBUTORS_LIMIT = 25 +TOP_CONTRIBUTORS_LIMIT = 15 class RepositoryContributor(BulkSaveModel, TimestampedModel): diff --git a/backend/apps/github/models/user.py b/backend/apps/github/models/user.py index 08ea2e2f7..70e0c2a3c 100644 --- a/backend/apps/github/models/user.py +++ b/backend/apps/github/models/user.py @@ -27,8 +27,19 @@ def __str__(self): def is_indexable(self): """Users to index.""" return ( - self.login != "ghost" and self.login not in Organization.get_logins() - ) # See https://github.com/ghost for more info. + self.login != "ghost" # See https://github.com/ghost for more info. + and self.login not in Organization.get_logins() + ) + + @property + def issues(self): + """Return user issues.""" + return self.created_issues.all() + + @property + def releases(self): + """Return user releases.""" + return self.created_releases.all() def from_github(self, gh_user): """Update instance based on GitHub user data.""" diff --git a/backend/apps/owasp/index/project.py b/backend/apps/owasp/index/project.py index 7cee8a6bb..fbd3bb929 100644 --- a/backend/apps/owasp/index/project.py +++ b/backend/apps/owasp/index/project.py @@ -19,6 +19,8 @@ class ProjectIndex(AlgoliaIndex, IndexBase): "idx_custom_tags", "idx_description", "idx_forks_count", + "idx_issues", + "idx_issues_count", "idx_key", "idx_languages", "idx_leaders", @@ -26,8 +28,10 @@ class ProjectIndex(AlgoliaIndex, IndexBase): "idx_level", "idx_name", "idx_organizations", - "idx_repository_descriptions", - "idx_repository_names", + "idx_releases", + "idx_releases_count", + "idx_repositories", + "idx_repositories_count", "idx_stars_count", "idx_summary", "idx_tags", @@ -43,7 +47,7 @@ class ProjectIndex(AlgoliaIndex, IndexBase): "filterOnly(idx_key)", "idx_name", "idx_tags", - "idx_repository_names", + "idx_repositories.name", ], "indexLanguages": ["en"], "customRanking": [ @@ -64,7 +68,7 @@ class ProjectIndex(AlgoliaIndex, IndexBase): ], "searchableAttributes": [ "unordered(idx_name)", - "unordered(idx_repository_descriptions, idx_repository_names)", + "unordered(idx_repositories.description, idx_repositories.name)", "unordered(idx_custom_tags, idx_languages, idx_tags, idx_topics)", "unordered(idx_description)", "unordered(idx_companies, idx_organizations)", diff --git a/backend/apps/owasp/models/mixins/project.py b/backend/apps/owasp/models/mixins/project.py index 0931c4aee..1cf591eea 100644 --- a/backend/apps/owasp/models/mixins/project.py +++ b/backend/apps/owasp/models/mixins/project.py @@ -3,6 +3,10 @@ from apps.common.utils import join_values from apps.owasp.models.mixins.common import GenericEntityMixin +ISSUES_LIMIT = 6 +RELEASES_LIMIT = 4 +REPOSITORIES_LIMIT = 4 + class ProjectIndexMixin(GenericEntityMixin): """Project index mixin.""" @@ -28,15 +32,44 @@ def idx_forks_count(self): return self.forks_count @property - def idx_languages(self): - """Return languages for indexing.""" - return self.languages + def idx_issues(self): + """Return issues for indexing.""" + return [ + { + "author": { + "avatar_url": i.author.avatar_url if i.author else "", + "key": i.author.login if i.author else "", + "name": i.author.name if i.author else "", + }, + "created_at": i.created_at.timestamp(), + "comments_count": i.comments_count, + "number": i.number, + "repository": { + "key": i.repository.key, + "owner_key": i.repository.owner.login, + }, + "title": i.title, + } + for i in self.open_issues.select_related("author").order_by("-created_at")[ + :ISSUES_LIMIT + ] + ] + + @property + def idx_issues_count(self): + """Return issues count for indexing.""" + return self.open_issues.count() @property def idx_key(self): """Return key for indexing.""" return self.key.replace("www-project-", "") + @property + def idx_languages(self): + """Return languages for indexing.""" + return self.languages + @property def idx_level(self): """Return level text value for indexing.""" @@ -58,31 +91,57 @@ def idx_organizations(self): return join_values(fields=(o.name for o in self.organizations.all())) @property - def idx_repository_descriptions(self): - """Return repository descriptions for indexing. - - Description of the default OWASP project repository + 4 most recently updated repositories. - """ - return [self.owasp_repository.description] + [ - repository.description - for repository in self.repositories.exclude(id=self.owasp_repository.id) - .exclude(description="") - .order_by("-updated_at")[:4] + def idx_releases(self): + """Return releases for indexing.""" + return [ + { + "author": { + "avatar_url": r.author.avatar_url if r.author else "", + "key": r.author.login if r.author else "", + "name": r.author.name if r.author else "", + }, + "is_pre_release": r.is_pre_release, + "name": r.name, + "published_at": r.published_at.timestamp(), + "repository": { + "key": r.repository.key, + "owner_key": r.repository.owner.login, + }, + "tag_name": r.tag_name, + } + for r in self.published_releases.select_related("author").order_by("-published_at")[ + :RELEASES_LIMIT + ] ] @property - def idx_repository_names(self): - """Return repository names for indexing. - - Name of the default OWASP project repository + 4 most recently updated repositories. - """ - return [self.owasp_repository.name] + [ - repository.name - for repository in self.repositories.exclude(id=self.owasp_repository.id) - .exclude(name="") - .order_by("-updated_at")[:4] + def idx_releases_count(self): + """Return releases count for indexing.""" + return self.published_releases.count() + + @property + def idx_repositories(self): + """Return repositories for indexing.""" + return [ + { + "contributors_count": r.contributors_count, + "description": r.description, + "forks_count": r.forks_count, + "key": r.key.lower(), + "latest_release": str(r.latest_release.summary) if r.latest_release else "", + "license": r.license, + "name": r.name, + "owner_key": r.owner.login.lower(), + "stars_count": r.stars_count, + } + for r in self.repositories.order_by("-stars_count")[:REPOSITORIES_LIMIT] ] + @property + def idx_repositories_count(self): + """Return repositories count for indexing.""" + return self.repositories.count() + @property def idx_stars_count(self): """Return stars count for indexing.""" diff --git a/backend/apps/owasp/models/project.py b/backend/apps/owasp/models/project.py index 30c2b0a79..9f6e09df7 100644 --- a/backend/apps/owasp/models/project.py +++ b/backend/apps/owasp/models/project.py @@ -8,6 +8,8 @@ from apps.common.models import BulkSaveModel, TimestampedModel from apps.common.utils import get_absolute_url from apps.core.models.prompt import Prompt +from apps.github.models.issue import Issue +from apps.github.models.release import Release from apps.owasp.models.common import GenericEntityModel, RepositoryBasedEntityModel from apps.owasp.models.managers.project import ActiveProjectManager from apps.owasp.models.mixins.project import ProjectIndexMixin @@ -152,6 +154,20 @@ def nest_url(self): """Get Nest URL for project.""" return get_absolute_url(f"projects/{self.nest_key}") + @property + def open_issues(self): + """Return open issues.""" + return Issue.open_issues.filter(repository__in=self.repositories.all()) + + @property + def published_releases(self): + """Return project releases.""" + return Release.objects.filter( + is_draft=False, + published_at__isnull=False, + repository__in=self.repositories.all(), + ) + def deactivate(self): """Deactivate project.""" self.is_active = False