add on demand package data collection for golang #596

minecode/collectors/bitbucket.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+
+
+"""
+Collect bitbucket packages from bitbucket registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def bitbucket_get_all_package_version_author(subset_path):
+    """
+    Return a list of all version numbers along with author for the package.
+    """
+    repo_tags = f"https://api.bitbucket.org/2.0/repositories/{subset_path}/refs/tags"
+    version_author_list = []
+    try:
+        while repo_tags:
+            response = requests.get(repo_tags)
+            response.raise_for_status()
+            data = response.json()
+            if data["size"] > 0:
+                # Get all available versions
+                for item in data["values"]:
+                    version = item["name"]
+                    author = ""
+                    if "target" in item and item["target"]:
+                        if "author" in item["target"] and item["target"]["author"]:
+                            if item["target"]["author"]["type"] == "author":
+                                author = item["target"]["author"]["user"]["display_name"]
+                    version_author_list.append((version, author))
+            # Handle pagination
+            repo_tags = data.get("next", None)
+        return version_author_list
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")

minecode/collectors/generic.py

@@ -100,7 +100,7 @@ def packagedata_from_dict(package_data):
     return PackageData.from_data(cleaned_package_data)
 
 
-def map_fetchcode_supported_package(package_url, pipelines, priority=0):
+def map_fetchcode_supported_package(package_url, pipelines, priority=0, from_go_lang=False):
     """
     Add a `package_url` supported by fetchcode to the PackageDB.
 
@@ -116,6 +116,9 @@ def map_fetchcode_supported_package(package_url, pipelines, priority=0):
         logger.error(error)
         return error
 
+    if from_go_lang:
+        packages[0].type = "golang"
+        packages[0].namespace = "github.com/" + packages[0].namespace
     package_data = packages[0].to_dict()
 
     # Remove obsolete Package fields see https://github.com/aboutcode-org/fetchcode/issues/108

minecode/collectors/github.py

@@ -13,6 +13,40 @@
 from minecode.collectors.generic import map_fetchcode_supported_package
 
 
+def github_get_all_versions(subset_path):
+    """
+    Fetch all versions (tags) from a GitHub repository using the API
+    Returns a list of all version tags in the repository
+    """
+    import requests
+
+    url = f"https://api.github.com/repos/{subset_path}/tags"
+    version_list = []
+    page = 1
+
+    while True:
+        response = requests.get(
+            url,
+            params={"page": page, "per_page": 100},  # Max 100 per page
+            headers={"Accept": "application/vnd.github.v3+json"},
+        )
+        response.raise_for_status()
+
+        data = response.json()
+        if not data:
+            break
+
+        for tag in data:
+            version_list.append(tag["name"])
+        page += 1
+
+        # Check if we've reached the last page
+        if "next" not in response.links:
+            break
+
+    return version_list
+
+
 # Indexing GitHub PURLs requires a GitHub API token.
 # Please add your GitHub API key to the `.env` file, for example: `GH_TOKEN=your-github-api`.
 @priority_router.route("pkg:github/.*")

minecode/collectors/gitlab.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import requests
+
+
+"""
+Collect gitlab packages from gitlab registries.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def gitlab_get_all_package_version_author(subset_path):
+    """
+    Return a list of all version numbers along with author and author email
+    for the package.
+    """
+    repo_tags = f"https://gitlab.com/api/v4/projects/{subset_path}/repository/tags"
+    try:
+        response = requests.get(repo_tags)
+        response.raise_for_status()
+        data = response.json()
+        version_author_list = []
+        # Get all available versions
+        for item in data:
+            version = item["name"]
+            author = item["commit"]["author_name"]
+            author_email = item["commit"]["author_email"]
+            version_author_list.append((version, author, author_email))
+        return version_author_list
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")