Feature/migrate scraper to GitHub md files (#2223)

* add properties and methods for github.md files

* use Github .md files files instead of scraper for urls, leaders, and audience

* Update code(add tests)

* Update code

* Update code

* add get_leaders_emails() and add name, email to entity_member model

* Update code

* add entity_member/leaders sync

* update regex

* add tests

* update code and fix sonar and cr issues

* Update code

* Update code

* Fix cspell

---------

Co-authored-by: Kate Golovanova <[email protected]>
Co-authored-by: Arkadii Yakovets <[email protected]>

Author: 3 people

backend/apps/owasp/admin/entity_member.py

@@ -17,17 +17,8 @@ class EntityMemberAdmin(admin.ModelAdmin):
 
     actions = ("approve_members",)
     autocomplete_fields = ("member",)
-    fields = (
-        "entity_type",
-        "entity_id",
-        "member",
-        "role",
-        "order",
-        "is_active",
-        "is_reviewed",
-        "description",
-    )
     list_display = (
+        "member_name",
         "member",
         "entity",
         "owasp_url",

backend/apps/owasp/management/commands/owasp_scrape_chapters.py

@@ -39,11 +39,15 @@ def handle(self, *args, **options) -> None:
                 chapter.deactivate()
                 continue
 
+            chapter.leaders_raw = chapter.get_leaders()
+            if leaders_emails := chapter.get_leaders_emails():
+                chapter.sync_leaders(leaders_emails)
+
             # Get related URLs.
             scraped_urls = sorted(
                 {
                     repository_url
-                    for url in set(scraper.get_urls())
+                    for url in set(chapter.get_urls())
                     if (
                         repository_url := normalize_url(
                             chapter.get_related_url(

backend/apps/owasp/management/commands/owasp_scrape_committees.py

@@ -39,11 +39,15 @@ def handle(self, *args, **options) -> None:
                 committee.deactivate()
                 continue
 
+            committee.leaders_raw = committee.get_leaders()
+            if leaders_emails := committee.get_leaders_emails():
+                committee.sync_leaders(leaders_emails)
+
             # Get related URLs.
             scraped_urls = sorted(
                 {
                     repository_url
-                    for url in set(scraper.get_urls())
+                    for url in set(committee.get_urls())
                     if (
                         repository_url := normalize_url(
                             committee.get_related_url(

backend/apps/owasp/management/commands/owasp_scrape_projects.py

@@ -51,13 +51,16 @@ def handle(self, *args, **options) -> None:
                 project.deactivate()
                 continue
 
-            project.audience = scraper.get_audience()
+            project.audience = project.get_audience()
+            project.leaders_raw = project.get_leaders()
+            if leaders_emails := project.get_leaders_emails():
+                project.sync_leaders(leaders_emails)
 
             # Get GitHub URLs.
             scraped_urls = sorted(
                 {
                     repository_url
-                    for url in set(scraper.get_urls(domain="github.com"))
+                    for url in set(project.get_urls(domain="github.com"))
                     if (repository_url := normalize_url(project.get_related_url(url)))
                     and repository_url not in {project.github_url, project.owasp_url}
                 }

backend/apps/owasp/migrations/0051_entitymember_member_email_entitymember_member_name_and_more.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.2.6 on 2025-09-06 10:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("github", "0035_alter_user_bio_alter_user_is_owasp_staff"),
+        ("owasp", "0050_alter_entitymember_role"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="entitymember",
+            name="member_email",
+            field=models.EmailField(blank=True, default="", max_length=254),
+        ),
+        migrations.AddField(
+            model_name="entitymember",
+            name="member_name",
+            field=models.CharField(default="", max_length=255),
+        ),
+        migrations.AlterField(
+            model_name="entitymember",
+            name="member",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="+",
+                to="github.user",
+            ),
+        ),
+    ]

backend/apps/owasp/migrations/0052_remove_entitymember_owasp_entit_member__6e516f_idx_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.2.6 on 2025-09-11 01:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("contenttypes", "0002_remove_content_type_name"),
+        ("owasp", "0051_entitymember_member_email_entitymember_member_name_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="entitymember",
+            name="owasp_entit_member__6e516f_idx",
+        ),
+        migrations.AlterUniqueTogether(
+            name="entitymember",
+            unique_together=set(),
+        ),
+        migrations.AlterField(
+            model_name="entitymember",
+            name="member_name",
+            field=models.CharField(max_length=255),
+        ),
+        migrations.AlterUniqueTogether(
+            name="entitymember",
+            unique_together={("entity_type", "entity_id", "member_name", "role")},
+        ),
+    ]

backend/apps/owasp/models/common.py

@@ -8,8 +8,10 @@
 from urllib.parse import urlparse
 
 import yaml
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
 
+from apps.common.models import BulkSaveModel
 from apps.common.open_ai import OpenAi
 from apps.github.constants import (
     GITHUB_REPOSITORY_RE,
@@ -18,6 +20,7 @@
 from apps.github.models.user import User
 from apps.github.utils import get_repository_file_content
 from apps.owasp.models.entity_member import EntityMember
+from apps.owasp.models.enums.project import AudienceChoices
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +104,16 @@ def index_md_url(self) -> str | None:
             else None
         )
 
+    @property
+    def info_md_url(self) -> str | None:
+        """Return entity's raw info.md GitHub URL."""
+        return (
+            "https://raw.githubusercontent.com/OWASP/"
+            f"{self.owasp_repository.key}/{self.owasp_repository.default_branch}/info.md"
+            if self.owasp_repository
+            else None
+        )
+
     @property
     def entity_leaders(self) -> models.QuerySet[User]:
         """Return entity's leaders."""
@@ -160,6 +173,21 @@ def generate_summary(self, prompt, open_ai=None, max_tokens=500):
         open_ai.set_max_tokens(max_tokens).set_prompt(prompt)
         self.summary = open_ai.complete() or ""
 
+    def get_audience(self):
+        """Get audience from info.md file on GitHub."""
+        content = get_repository_file_content(self.info_md_url)
+        if not content:
+            return []
+
+        found_keywords = set()
+
+        for line in content.split("\n"):
+            for lower_kw, original_kw in AudienceChoices.choices:
+                if original_kw in line:
+                    found_keywords.add(lower_kw)
+
+        return sorted(found_keywords)
+
     def get_leaders(self):
         """Get leaders from leaders.md file on GitHub."""
         content = get_repository_file_content(self.leaders_md_url)
@@ -182,6 +210,26 @@ def get_leaders(self):
 
         return leaders
 
+    def get_leaders_emails(self):
+        """Get leaders emails from leaders.md file on GitHub."""
+        content = get_repository_file_content(self.leaders_md_url)
+        if not content:
+            return {}
+
+        leaders = {}
+        for line in content.split("\n"):
+            matches = re.findall(
+                r"^[-*]\s*\[([^\]]+)\]\(mailto:([^)]+)(\)|([^[<\n]))", line.strip()
+            )
+
+            for match in matches:
+                if match[0] and match[1]:  # Name with email
+                    leaders[match[0].strip()] = match[1].strip()
+                elif match[2]:  # Name without email
+                    leaders[match[2].strip()] = None
+
+        return leaders
+
     def get_metadata(self):
         """Get entity metadata."""
         try:
@@ -224,6 +272,19 @@ def get_related_url(self, url, exclude_domains=(), include_domains=()) -> str |
 
         return url
 
+    def get_urls(self, domain=None):
+        """Get URLs from info.md file on GitHub."""
+        content = get_repository_file_content(self.info_md_url)
+        if not content:
+            return []
+
+        urls = re.findall(r"https?:\/\/[^\s\)]+", content.strip())
+
+        if domain:
+            return [url for url in urls if urlparse(url).netloc == domain]
+
+        return urls
+
     def parse_tags(self, tags) -> list[str]:
         """Parse entity tags."""
         if not tags:
@@ -234,3 +295,33 @@ def parse_tags(self, tags) -> list[str]:
             if isinstance(tags, str)
             else tags
         )
+
+    def sync_leaders(self, leaders_emails):
+        """Sync Leaders data.
+
+        Args:
+            leaders_emails (dict[str, str | None]): A dictionary
+            where keys are the full names of the leaders
+            and values are their corresponding email addresses (or None if no email is provided).
+
+        """
+        content_type = ContentType.objects.get_for_model(self.__class__)
+
+        leaders = []
+        for order, (name, email) in enumerate(leaders_emails.items()):
+            leaders.append(
+                EntityMember.update_data(
+                    {
+                        "entity_id": self.id,
+                        "entity_type": content_type,
+                        "member_email": email or "",
+                        "member_name": name,
+                        "order": (order + 1) * 100,
+                        "role": EntityMember.Role.LEADER,
+                    },
+                    save=False,
+                )
+            )
+
+        if leaders:
+            BulkSaveModel.bulk_save(EntityMember, leaders)

backend/apps/owasp/models/entity_member.py

@@ -20,12 +20,11 @@ class Meta:
         unique_together = (
             "entity_type",
             "entity_id",
-            "member",
+            "member_name",
             "role",
         )
         indexes = [
             models.Index(fields=["entity_type", "entity_id"]),
-            models.Index(fields=["member"]),
         ]
         verbose_name_plural = "Entity members"
 
@@ -35,10 +34,6 @@ class Meta:
         help_text="Optional note or role description",
         max_length=100,
     )
-    entity = GenericForeignKey("entity_type", "entity_id")
-    entity_id = models.PositiveBigIntegerField()
-    entity_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
-
     is_active = models.BooleanField(
         default=False,
         help_text="Indicates if the membership is active",
@@ -47,11 +42,8 @@ class Meta:
         default=False,
         help_text="Indicates if the membership is reviewed",
     )
-    member = models.ForeignKey(
-        User,
-        on_delete=models.CASCADE,
-        related_name="+",
-    )
+    member_email = models.EmailField(blank=True, default="")
+    member_name = models.CharField(max_length=255)
     order = models.PositiveSmallIntegerField(
         default=0,
         help_text="Display order/priority of members",
@@ -62,6 +54,59 @@ class Meta:
         default=Role.LEADER,
     )
 
+    # FKs.
+    member = models.ForeignKey(
+        User,
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="+",
+    )
+
+    # GFKs.
+    entity = GenericForeignKey("entity_type", "entity_id")
+    entity_id = models.PositiveBigIntegerField()
+    entity_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+
     def __str__(self):
         """EntityMember human readable representation."""
-        return f"{self.member.login} as {self.get_role_display()} for {self.entity}"
+        display_name = self.member.login if self.member else self.member_name
+        return f"{display_name} as {self.get_role_display()} for {self.entity}"
+
+    @staticmethod
+    def update_data(data, *, save: bool = True) -> "EntityMember":
+        """Update entity member data."""
+        try:
+            entity_member = EntityMember.objects.get(
+                entity_id=data["entity_id"],
+                entity_type=data["entity_type"],
+                member_name=data["member_name"],
+                role=data["role"],
+            )
+        except EntityMember.DoesNotExist:
+            entity_member = EntityMember(
+                entity_id=data["entity_id"],
+                entity_type=data["entity_type"],
+                member_name=data["member_name"],
+                role=data["role"],
+            )
+
+        entity_member.from_dict(data)
+        if save:
+            entity_member.save()
+
+        return entity_member
+
+    def from_dict(self, data) -> None:
+        """Update instance based on dict data."""
+        fields = {
+            "entity_id": data["entity_id"],
+            "entity_type": data["entity_type"],
+            "member_email": data.get("member_email", ""),
+            "member_name": data["member_name"],
+            "order": data.get("order", 0),
+            "role": data["role"],
+        }
+
+        for key, value in fields.items():
+            setattr(self, key, value)

backend/tests/apps/owasp/management/commands/owasp_scrape_chapters_test.py

@@ -38,7 +38,7 @@ def mock_chapter(self):
     @mock.patch.object(Chapter, "bulk_save", autospec=True)
     def test_handle(self, mock_bulk_save, command, mock_chapter, offset, chapters):
         mock_scraper = mock.Mock(spec=OwaspScraper)
-        mock_scraper.get_urls.return_value = [
+        mock_chapter.get_urls.return_value = [
             "https://example.com/repo1",
             "https://example.com/repo2",
             "https://invalid.com/repo3",

backend/tests/apps/owasp/management/commands/owasp_scrape_committees_test.py

@@ -38,7 +38,7 @@ def mock_committee(self):
     @mock.patch.object(Committee, "bulk_save", autospec=True)
     def test_handle(self, mock_bulk_save, command, mock_committee, offset, committees):
         mock_scraper = mock.Mock(spec=OwaspScraper)
-        mock_scraper.get_urls.return_value = [
+        mock_committee.get_urls.return_value = [
             "https://example.com/repo1",
             "https://example.com/repo2",
             "https://invalid.com/repo3",